Merge branch 'neo' into master
Change-Id: I85fa54a43491e45b30c88b0ea880fa093dccd08e
diff --git a/asterixdb/asterix-active/pom.xml b/asterixdb/asterix-active/pom.xml
index 971534f..f5b1978 100644
--- a/asterixdb/asterix-active/pom.xml
+++ b/asterixdb/asterix-active/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-active</artifactId>
<properties>
diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml
index f542af3..7a24cbe 100644
--- a/asterixdb/asterix-algebra/pom.xml
+++ b/asterixdb/asterix-algebra/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-algebra</artifactId>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index 06a6687..a8dcb1f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -21,14 +21,18 @@
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.DataSourceIndex;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.optimizer.rules.am.BTreeJobGenParams;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -45,6 +49,7 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator;
@@ -62,9 +67,13 @@
import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
/**
* Contributes the runtime operator for an unnest-map representing a BTree search.
@@ -131,6 +140,7 @@
long outputLimit = -1;
boolean retainMissing = false;
IMissingWriterFactory nonMatchWriterFactory = null;
+ ITupleProjectorFactory tupleProjectorFactory = DefaultTupleProjectorFactory.INSTANCE;
switch (unnestMap.getOperatorTag()) {
case UNNEST_MAP:
UnnestMapOperator unnestMapOp = (UnnestMapOperator) unnestMap;
@@ -139,6 +149,15 @@
tupleFilterFactory = metadataProvider.createTupleFilterFactory(new IOperatorSchema[] { opSchema },
typeEnv, unnestMapOp.getSelectCondition().getValue(), context);
}
+ DatasetFormatInfo formatInfo = dataset.getDatasetFormatInfo();
+ if (isPrimaryIndex && formatInfo.getFormat() == DatasetConfig.DatasetFormat.COLUMN) {
+ IProjectionFiltrationInfo<?> projectionInfo = unnestMapOp.getDatasetProjectionInfo();
+ IProjectionFiltrationInfo<?> metaProjectionInfo = unnestMapOp.getMetaProjectionInfo();
+ ARecordType datasetType = (ARecordType) metadataProvider.findType(dataset);
+ ARecordType metaItemType = (ARecordType) metadataProvider.findMetaType(dataset);
+ tupleProjectorFactory = IndexUtil.createTupleProjectorFactory(formatInfo, projectionInfo,
+ metaProjectionInfo, datasetType, metaItemType, dataset.getPrimaryKeys().size());
+ }
break;
case LEFT_OUTER_UNNEST_MAP:
// By nature, LEFT_OUTER_UNNEST_MAP should generate missing (or null) values for non-matching tuples.
@@ -157,7 +176,8 @@
nonMatchWriterFactory, dataset, jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes,
jobGenParams.isLowKeyInclusive(), jobGenParams.isHighKeyInclusive(), propagateFilter,
nonFilterWriterFactory, minFilterFieldIndexes, maxFilterFieldIndexes, tupleFilterFactory, outputLimit,
- unnestMap.getGenerateCallBackProceedResultVar(), isPrimaryIndexPointSearch(op));
+ unnestMap.getGenerateCallBackProceedResultVar(),
+ isPrimaryIndexPointSearch(op, context.getPhysicalOptimizationConfig()), tupleProjectorFactory);
IOperatorDescriptor opDesc = btreeSearch.first;
opDesc.setSourceLocation(unnestMap.getSourceLocation());
@@ -168,8 +188,12 @@
builder.contributeGraphEdge(srcExchange, 0, unnestMap, 0);
}
- private boolean isPrimaryIndexPointSearch(ILogicalOperator op) {
- if (!isEqCondition || !isPrimaryIndex || !lowKeyVarList.equals(highKeyVarList)) {
+ /**
+ * Check whether we can use {@link LSMBTreeBatchPointSearchCursor} to perform point-lookups on the primary index
+ */
+ private boolean isPrimaryIndexPointSearch(ILogicalOperator op, PhysicalOptimizationConfig config) {
+ if (!config.isBatchLookupEnabled() || !isEqCondition || !isPrimaryIndex
+ || !lowKeyVarList.equals(highKeyVarList)) {
return false;
}
Index searchIndex = ((DataSourceIndex) idx).getIndex();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
index 4ad888c..b4eab4d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
@@ -85,12 +85,12 @@
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, CompilerProperties.COMPILER_ARRAYINDEX_KEY,
CompilerProperties.COMPILER_CBO_KEY, CompilerProperties.COMPILER_CBO_TEST_KEY,
CompilerProperties.COMPILER_FORCE_JOIN_ORDER_KEY, CompilerProperties.COMPILER_QUERY_PLAN_SHAPE_KEY,
- CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
- FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
- StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
- FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
- SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
- DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
+ CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, CompilerProperties.COMPILER_COLUMN_FILTER_KEY,
+ FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME,
+ FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION,
+ FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS,
+ SqlppQueryRewriter.INLINE_WITH_OPTION, SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION,
+ "hash_merge", "output-record-type", DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN,
EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK, SqlppQueryRewriter.SQL_COMPAT_OPTION));
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
index b1d0b47..b8685d5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
@@ -26,7 +26,6 @@
import org.apache.asterix.external.library.ExternalFunctionDescriptorProvider;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionManager;
import org.apache.asterix.om.functions.IFunctionTypeInferer;
@@ -138,7 +137,7 @@
throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
IFunctionDescriptor fd;
- if (expr.getFunctionInfo() instanceof IExternalFunctionInfo) {
+ if (expr.getFunctionInfo().isExternal()) {
// Expr is an external function
fd = ExternalFunctionDescriptorProvider.resolveExternalFunction(expr, env, context);
} else {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 1c3ef03..e63b865 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -79,7 +79,7 @@
import org.apache.asterix.optimizer.rules.PushLimitIntoPrimarySearchRule;
import org.apache.asterix.optimizer.rules.PushProperJoinThroughProduct;
import org.apache.asterix.optimizer.rules.PushSimilarityFunctionsBelowJoin;
-import org.apache.asterix.optimizer.rules.PushValueAccessToExternalDataScanRule;
+import org.apache.asterix.optimizer.rules.PushValueAccessToDataScanRule;
import org.apache.asterix.optimizer.rules.RemoveDuplicateFieldsRule;
import org.apache.asterix.optimizer.rules.RemoveLeftOuterUnnestForLeftOuterJoinRule;
import org.apache.asterix.optimizer.rules.RemoveOrReplaceDefaultNullCastRule;
@@ -172,7 +172,7 @@
return translationRules;
}
- public static final List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() {
List<IAlgebraicRewriteRule> typeInfer = new LinkedList<>();
typeInfer.add(new InlineUnnestFunctionRule());
typeInfer.add(new InferTypesRule());
@@ -180,17 +180,17 @@
return typeInfer;
}
- public static final List<IAlgebraicRewriteRule> buildAutogenerateIDRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildAutogenerateIDRuleCollection() {
List<IAlgebraicRewriteRule> autogen = new LinkedList<>();
autogen.add(new IntroduceAutogenerateIDRule());
return autogen;
}
- public static final List<IAlgebraicRewriteRule> buildFulltextContainsRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildFulltextContainsRuleCollection() {
return Collections.singletonList(new FullTextContainsParameterCheckAndSetRule());
}
- public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection(ICcApplicationContext appCtx) {
+ public static List<IAlgebraicRewriteRule> buildNormalizationRuleCollection(ICcApplicationContext appCtx) {
List<IAlgebraicRewriteRule> normalization = new LinkedList<>();
normalization.add(new CheckInsertUpsertReturningRule());
normalization.add(new IntroduceUnnestForCollectionToSequenceRule());
@@ -228,7 +228,7 @@
return normalization;
}
- public static final List<IAlgebraicRewriteRule> buildCondPushDownAndJoinInferenceRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildCondPushDownAndJoinInferenceRuleCollection() {
List<IAlgebraicRewriteRule> condPushDownAndJoinInference = new LinkedList<>();
condPushDownAndJoinInference.add(new PushSelectDownRule());
@@ -271,7 +271,7 @@
return condPushDownAndJoinInference;
}
- public static final List<IAlgebraicRewriteRule> buildLoadFieldsRuleCollection(ICcApplicationContext appCtx) {
+ public static List<IAlgebraicRewriteRule> buildLoadFieldsRuleCollection(ICcApplicationContext appCtx) {
List<IAlgebraicRewriteRule> fieldLoads = new LinkedList<>();
fieldLoads.add(new LoadRecordFieldsRule());
fieldLoads.add(new PushFieldAccessRule());
@@ -292,7 +292,7 @@
return fieldLoads;
}
- public static final List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() {
List<IAlgebraicRewriteRule> consolidation = new LinkedList<>();
consolidation.add(new ConsolidateSelectsRule());
consolidation.add(new ConsolidateAssignsRule(false));
@@ -317,7 +317,7 @@
return consolidation;
}
- public static final List<IAlgebraicRewriteRule> buildAccessMethodRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildAccessMethodRuleCollection() {
List<IAlgebraicRewriteRule> accessMethod = new LinkedList<>();
accessMethod.add(new IntroduceSelectAccessMethodRule());
accessMethod.add(new IntroduceJoinAccessMethodRule());
@@ -330,7 +330,7 @@
return accessMethod;
}
- public static final List<IAlgebraicRewriteRule> buildPlanCleanupRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildPlanCleanupRuleCollection() {
List<IAlgebraicRewriteRule> planCleanupRules = new LinkedList<>();
planCleanupRules.add(new SwitchInnerJoinBranchRule());
planCleanupRules.add(new AsterixPushMapOperatorThroughUnionRule(LogicalOperatorTag.ASSIGN));
@@ -364,14 +364,14 @@
return planCleanupRules;
}
- public static final List<IAlgebraicRewriteRule> buildDataExchangeRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildDataExchangeRuleCollection() {
List<IAlgebraicRewriteRule> dataExchange = new LinkedList<>();
dataExchange.add(new SetExecutionModeRule());
dataExchange.add(new FindDataSourcesRule());
return dataExchange;
}
- public static final List<IAlgebraicRewriteRule> buildCBORuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildCBORuleCollection() {
List<IAlgebraicRewriteRule> cbo = new LinkedList<>();
cbo.add(new ConsolidateSelectsRule());
cbo.add(new EnumerateJoinsRule(new JoinEnum()));
@@ -380,7 +380,7 @@
return cbo;
}
- public static final List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() {
+ public static List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() {
List<IAlgebraicRewriteRule> physicalRewritesAllLevels = new LinkedList<>();
physicalRewritesAllLevels.add(new PullSelectOutOfEqJoin());
physicalRewritesAllLevels.add(new ExtractBatchableExternalFunctionCallsRule());
@@ -408,7 +408,7 @@
return physicalRewritesAllLevels;
}
- public static final List<IAlgebraicRewriteRule> buildPhysicalRewritesTopLevelRuleCollection(
+ public static List<IAlgebraicRewriteRule> buildPhysicalRewritesTopLevelRuleCollection(
ICcApplicationContext appCtx) {
List<IAlgebraicRewriteRule> physicalRewritesTopLevel = new LinkedList<>();
physicalRewritesTopLevel.add(new PushNestedOrderByUnderPreSortedGroupByRule());
@@ -418,11 +418,17 @@
physicalRewritesTopLevel.add(new ConstantFoldingRule(appCtx));
physicalRewritesTopLevel.add(new PushLimitIntoOrderByRule());
//Must run before PushLimitIntoPrimarySearchRule to ensure the select condition is inspected
- physicalRewritesTopLevel.add(new PushValueAccessToExternalDataScanRule());
physicalRewritesTopLevel.add(new PushLimitIntoPrimarySearchRule());
// remove assigns that could become unused after PushLimitIntoPrimarySearchRule
physicalRewritesTopLevel.add(new RemoveUnusedAssignAndAggregateRule());
physicalRewritesTopLevel.add(new IntroduceProjectsRule());
+ //Infer the types for the pushed down condition
+ physicalRewritesTopLevel.add(new InferTypesRule());
+ /*
+ * Must run IntroduceProjectsRule before PushValueAccessToDataScanRule to ensure that no entire records are
+ * returned if they are projected out
+ */
+ physicalRewritesTopLevel.add(new PushValueAccessToDataScanRule());
physicalRewritesTopLevel.add(new SetAsterixPhysicalOperatorsRule());
physicalRewritesTopLevel.add(new IntroduceRapidFrameFlushProjectAssignRule());
physicalRewritesTopLevel.add(new SetExecutionModeRule());
@@ -430,7 +436,7 @@
return physicalRewritesTopLevel;
}
- public static final List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() {
+ public static List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() {
List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<>();
prepareForJobGenRewrites.add(new InsertProjectBeforeUnionRule());
prepareForJobGenRewrites.add(new SetAsterixPhysicalOperatorsRule());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index fe000c3..9fe1ba4 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -47,7 +47,6 @@
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
@@ -393,7 +392,7 @@
private boolean canConstantFold(ScalarFunctionCallExpression function) throws AlgebricksException {
// skip external functions because they're not available at compile time (on CC)
IFunctionInfo fi = function.getFunctionInfo();
- if (fi instanceof IExternalFunctionInfo) {
+ if (fi.isExternal()) {
return false;
}
// skip all functions that would produce records/arrays/multisets (derived types) in their open format
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessToExternalDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessToDataScanRule.java
similarity index 91%
rename from asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessToExternalDataScanRule.java
rename to asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessToDataScanRule.java
index 405e2bd..d43a5d1 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessToExternalDataScanRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessToDataScanRule.java
@@ -63,7 +63,7 @@
* The resulting record $$r will be {"personalInfo":{"age": *AGE*}, "salary": *SALARY*}
* and other fields will not be included in $$r.
*/
-public class PushValueAccessToExternalDataScanRule implements IAlgebraicRewriteRule {
+public class PushValueAccessToDataScanRule implements IAlgebraicRewriteRule {
//Initially, assume we need to run the rule
private boolean run = true;
@@ -76,7 +76,7 @@
}
/*
- * Only run the rewrite rule once and only if the plan contains a data-scan on an external dataset that
+ * Only run the rewrite rule once and only if the plan contains a data-scan on a dataset that
* support value access pushdown.
*/
run = shouldRun(context);
@@ -92,7 +92,7 @@
}
/**
- * Check whether the plan contains an external dataset that supports pushdown
+ * Check whether the plan contains a dataset that supports pushdown
*
* @param context optimization context
* @return true if the plan contains such dataset, false otherwise
@@ -117,7 +117,8 @@
String datasetName = dataSource.getId().getDatasourceName();
Dataset dataset = metadataProvider.findDataset(dataverse, datasetName);
- return dataset != null && dataset.getDatasetType() == DatasetConfig.DatasetType.EXTERNAL && ExternalDataUtils
- .supportsPushdown(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties());
+ return dataset != null && ((dataset.getDatasetType() == DatasetConfig.DatasetType.EXTERNAL && ExternalDataUtils
+ .supportsPushdown(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()))
+ || dataset.getDatasetFormatInfo().getFormat() == DatasetConfig.DatasetFormat.COLUMN);
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 2509010..2a6604e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -61,6 +61,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -205,6 +206,11 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return null;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 25d42df..9c69902 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -26,12 +26,15 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.asterix.algebra.operators.physical.ExternalDataLookupPOperator;
import org.apache.asterix.common.annotations.AbstractExpressionAnnotationWithIndexNames;
@@ -86,6 +89,7 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
@@ -94,6 +98,7 @@
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
@@ -148,6 +153,19 @@
BuiltinFunctions.YEAR_MONTH_DURATION_DEFAULT_NULL_CONSTRUCTOR,
BuiltinFunctions.UUID_DEFAULT_NULL_CONSTRUCTOR, BuiltinFunctions.BINARY_BASE64_DEFAULT_NULL_CONSTRUCTOR);
+ // TODO (GLENN): We can definitely expand the whitelist here...
+ private final static Map<FunctionIdentifier, Set<Integer>> INDEX_USE_ON_FUNCTION_CALL_WHITELIST;
+ private final static Set<Integer> ALL_INDEX_FUNCTION_ARGUMENTS = Collections.emptySet();
+ static {
+ INDEX_USE_ON_FUNCTION_CALL_WHITELIST = new HashMap<>();
+ INDEX_USE_ON_FUNCTION_CALL_WHITELIST.put(BuiltinFunctions.RECORD_ADD, Set.of(0));
+ INDEX_USE_ON_FUNCTION_CALL_WHITELIST.put(BuiltinFunctions.ADD_FIELDS, Set.of(0));
+ INDEX_USE_ON_FUNCTION_CALL_WHITELIST.put(BuiltinFunctions.RECORD_REMOVE, Set.of(0));
+ INDEX_USE_ON_FUNCTION_CALL_WHITELIST.put(BuiltinFunctions.RECORD_RENAME, Set.of(0));
+ INDEX_USE_ON_FUNCTION_CALL_WHITELIST.put(BuiltinFunctions.REMOVE_FIELDS, Set.of(0));
+ INDEX_USE_ON_FUNCTION_CALL_WHITELIST.put(BuiltinFunctions.RECORD_CONCAT, ALL_INDEX_FUNCTION_ARGUMENTS);
+ }
+
private final static Pair<List<String>, Integer> NO_FIELD_NAME = new Pair<>(Collections.emptyList(), 0);
public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, IAType metaItemType,
@@ -2989,8 +3007,26 @@
isByName = true;
}
if (isFieldAccess) {
- LogicalVariable sourceVar =
- ((VariableReferenceExpression) funcExpr.getArguments().get(0).getValue()).getVariableReference();
+ ILogicalExpression funcExprArg0 = funcExpr.getArguments().get(0).getValue();
+ MutableInt sourceIndicator = new MutableInt(0);
+ LogicalVariable sourceVar;
+ if (funcExprArg0.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ // This might be a field-access on an indexable-function-call (or nested indexable-function-calls).
+ List<LogicalVariable> foundDatasourceVariables = new ArrayList<>();
+ if (canUseIndexOnFunction((AbstractFunctionCallExpression) funcExprArg0, sourceIndicator,
+ foundDatasourceVariables, optFuncExpr, op.computeInputTypeEnvironment(context), context)) {
+ // TODO (GLENN): In the case of OBJECT_CONCAT w/ potentially multiple datasource variables, we
+ // will not explore each variable. This method definitely needs refactoring in the
+ // future to handle such a case.
+ sourceVar = foundDatasourceVariables.get(0);
+ } else {
+ return NO_FIELD_NAME;
+ }
+ } else if (funcExprArg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return NO_FIELD_NAME;
+ } else {
+ sourceVar = ((VariableReferenceExpression) funcExprArg0).getVariableReference();
+ }
if (optFuncExpr != null) {
optFuncExpr.setLogicalExpr(funcVarIndex, parentFuncExpr);
optFuncExpr.addStepExpr(funcVarIndex, funcExpr);
@@ -3028,12 +3064,27 @@
if (assignAndExpressionIndexes != null && assignAndExpressionIndexes[0] > -1) {
//We found the nested assign
- //Recursive call on nested assign
- Pair<List<String>, Integer> parentFieldNames =
- getFieldNameAndStepsFromSubTree(optFuncExpr, subTree, assignAndExpressionIndexes[0],
- assignAndExpressionIndexes[1], funcVarIndex, parentFuncExpr, context);
+ // Is the next operator composed of functions that are not a field access? If so, do not recurse.
+ ILogicalOperator nextOp = subTree.getAssignsAndUnnests().get(assignAndExpressionIndexes[0]);
+ boolean isIndexOnFunction = false;
+ if (nextOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator nextAssignOp = (AssignOperator) nextOp;
+ ILogicalExpression leadingArgumentExpr = nextAssignOp.getExpressions().get(0).getValue();
+ if (leadingArgumentExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ IVariableTypeEnvironment typeEnv = nextAssignOp.computeInputTypeEnvironment(context);
+ isIndexOnFunction = canUseIndexOnFunction((AbstractFunctionCallExpression) leadingArgumentExpr,
+ sourceIndicator, new HashSet<>(), optFuncExpr, typeEnv, context);
+ }
+ }
- if (parentFieldNames.first.isEmpty()) {
+ // Otherwise... recurse.
+ Pair<List<String>, Integer> parentFieldNames =
+ !isIndexOnFunction
+ ? getFieldNameAndStepsFromSubTree(optFuncExpr, subTree, assignAndExpressionIndexes[0],
+ assignAndExpressionIndexes[1], funcVarIndex, parentFuncExpr, context)
+ : NO_FIELD_NAME;
+
+ if (parentFieldNames.first.isEmpty() && !isIndexOnFunction) {
//Nested assign was not a field access.
//We will not use index
return NO_FIELD_NAME;
@@ -3056,13 +3107,22 @@
optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex));
}
- //add fieldName to the nested fieldName, return
- if (nestedAccessFieldName != null) {
- parentFieldNames.first.addAll(nestedAccessFieldName);
+ if (!isIndexOnFunction) {
+ //add fieldName to the nested fieldName, return
+ if (nestedAccessFieldName != null) {
+ parentFieldNames.first.addAll(nestedAccessFieldName);
+ } else {
+ parentFieldNames.first.add(fieldName);
+ }
+ return (parentFieldNames);
+
} else {
- parentFieldNames.first.add(fieldName);
+ if (nestedAccessFieldName != null) {
+ return new Pair<>(nestedAccessFieldName, sourceIndicator.getValue());
+ } else {
+ return new Pair<>(new ArrayList<>(List.of(fieldName)), sourceIndicator.getValue());
+ }
}
- return (parentFieldNames);
}
if (optFuncExpr != null) {
@@ -3199,4 +3259,73 @@
return funId.equals(FIELD_ACCESS_BY_NAME) || funId.equals(FIELD_ACCESS_BY_INDEX)
|| funId.equals(FIELD_ACCESS_NESTED);
}
+
+ /**
+ * If we are accessing some field through a function application (or series of function applications) of the
+ * following:
+ * <p><pre>
+ * | OBJECT_ADD | OBJECT_REMOVE | OBJECT_ADD_FIELDS |
+ * | OBJECT_CONCAT | OBJECT_RENAME | OBJECT_REMOVE_FIELDS |
+ * </pre>
+ * ...then we still might be able to use an index. Check the output type of applying our function(s) and verify
+ * that the input is a data source variable.
+ */
+ public static boolean canUseIndexOnFunction(AbstractFunctionCallExpression funcExpr, MutableInt sourceIndicator,
+ Collection<LogicalVariable> foundDatasourceVariables, IOptimizableFuncExpr optFuncExpr,
+ IVariableTypeEnvironment typeEnv, IOptimizationContext context) throws AlgebricksException {
+ FunctionIdentifier functionID = funcExpr.getFunctionIdentifier();
+ if (!INDEX_USE_ON_FUNCTION_CALL_WHITELIST.containsKey(functionID)) {
+ return false;
+ }
+
+ // Our output should be an object (this is more of a sanity check given that we have a whitelist).
+ IExpressionTypeComputer expressionTypeComputer = context.getExpressionTypeComputer();
+ IMetadataProvider<?, ?> metadataProvider = context.getMetadataProvider();
+ IAType originalOutputType = (IAType) expressionTypeComputer.getType(funcExpr, metadataProvider, typeEnv);
+ IAType outputType = TypeComputeUtils.getActualType(originalOutputType);
+ ARecordType outputRecordType = TypeComputeUtils.extractRecordType(outputType);
+ if (outputRecordType == null) {
+ return false;
+ }
+
+ // Check the type of our input, according to record variables in each function's argument.
+ boolean isDataSourceVariableFound = false;
+ Set<Integer> indicesToCheck = INDEX_USE_ON_FUNCTION_CALL_WHITELIST.get(functionID);
+ if (indicesToCheck.equals(ALL_INDEX_FUNCTION_ARGUMENTS)) {
+ indicesToCheck = IntStream.range(0, funcExpr.getArguments().size()).boxed().collect(Collectors.toSet());
+ }
+ for (Integer functionCallArgumentIndex : indicesToCheck) {
+ ILogicalExpression inputRecordExpr = funcExpr.getArguments().get(functionCallArgumentIndex).getValue();
+ switch (inputRecordExpr.getExpressionTag()) {
+ case FUNCTION_CALL:
+ AbstractFunctionCallExpression arg0FuncExpr = (AbstractFunctionCallExpression) inputRecordExpr;
+ isDataSourceVariableFound |= canUseIndexOnFunction(arg0FuncExpr, sourceIndicator,
+ foundDatasourceVariables, optFuncExpr, typeEnv, context);
+ break;
+
+ case VARIABLE:
+ // Base case. We should be using a data source variable here.
+ VariableReferenceExpression inputRecordVarExpr = (VariableReferenceExpression) inputRecordExpr;
+ LogicalVariable inputRecordVar = inputRecordVarExpr.getVariableReference();
+ if (optFuncExpr != null) {
+ for (int i = 0; i < optFuncExpr.getNumLogicalVars(); i++) {
+ OptimizableOperatorSubTree operatorSubTree = optFuncExpr.getOperatorSubTree(i);
+ if (operatorSubTree == null) {
+ continue;
+ }
+ if (operatorSubTree.getDataSourceVariables().stream().anyMatch(inputRecordVar::equals)) {
+ OptimizableOperatorSubTree.RecordTypeSource recordTypeSource =
+ operatorSubTree.getRecordTypeFor(inputRecordVar);
+ foundDatasourceVariables.add(inputRecordVar);
+ sourceIndicator.setValue(recordTypeSource.sourceIndicator);
+ isDataSourceVariableFound = true;
+ break;
+ }
+ }
+ }
+ break;
+ }
+ }
+ return isDataSourceVariableFound;
+ }
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/AbstractOperatorFromSubplanRewrite.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/AbstractOperatorFromSubplanRewrite.java
index 530596d..3328127 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/AbstractOperatorFromSubplanRewrite.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/AbstractOperatorFromSubplanRewrite.java
@@ -537,9 +537,7 @@
if (splitIntoConjuncts(conjunct.getValue(), innerExprConjuncts)) {
conjuncts.addAll(innerExprConjuncts);
} else {
-
conjuncts.add(new MutableObject<>(conjunct.getValue()));
-
}
}
return true;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EstimatedCostComputationVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EstimatedCostComputationVisitor.java
index 3943cf4..822d824 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EstimatedCostComputationVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EstimatedCostComputationVisitor.java
@@ -54,6 +54,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -169,6 +170,11 @@
}
@Override
+ public Pair<Double, Double> visitSwitchOperator(SwitchOperator op, Double arg) throws AlgebricksException {
+ return annotate(this, op, arg);
+ }
+
+ @Override
public Pair<Double, Double> visitMaterializeOperator(MaterializeOperator op, Double arg)
throws AlgebricksException {
return annotate(this, op, arg);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaBuilder.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaBuilder.java
index b7632db..f972677 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaBuilder.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaBuilder.java
@@ -21,7 +21,9 @@
import static org.apache.asterix.optimizer.rules.pushdown.ExpressionValueAccessPushdownVisitor.ARRAY_FUNCTIONS;
import static org.apache.asterix.optimizer.rules.pushdown.ExpressionValueAccessPushdownVisitor.SUPPORTED_FUNCTIONS;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.asterix.om.functions.BuiltinFunctions;
@@ -35,12 +37,15 @@
import org.apache.asterix.optimizer.rules.pushdown.schema.ObjectExpectedSchemaNode;
import org.apache.asterix.optimizer.rules.pushdown.schema.RootExpectedSchemaNode;
import org.apache.asterix.optimizer.rules.pushdown.schema.UnionExpectedSchemaNode;
-import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
@@ -53,31 +58,44 @@
* 2- the output type of getField("hashtags") is ARRAY
* 3- the output type of getItem(0) is ANY node
*/
-class ExpectedSchemaBuilder {
+public class ExpectedSchemaBuilder {
//Registered Variables
private final Map<LogicalVariable, IExpectedSchemaNode> varToNode;
- private final ExpectedSchemaNodeToIATypeTranslatorVisitor typeBuilder;
+ //Inferred type for expressions
+ private final Map<AbstractFunctionCallExpression, IExpectedSchemaNode> exprToNode;
public ExpectedSchemaBuilder() {
varToNode = new HashMap<>();
- typeBuilder = new ExpectedSchemaNodeToIATypeTranslatorVisitor();
+ exprToNode = new HashMap<>();
}
- public DataProjectionInfo createProjectionInfo(LogicalVariable recordVariable) {
+ public DataProjectionFiltrationInfo createProjectionInfo(LogicalVariable recordVariable) {
+ return createProjectionInfo(recordVariable, Collections.emptyMap(), null, null);
+ }
+
+ public DataProjectionFiltrationInfo createProjectionInfo(LogicalVariable recordVariable,
+ Map<ILogicalExpression, ARecordType> filterPaths, ILogicalExpression filterExpression,
+ Map<String, FunctionCallInformation> sourceInformationMap) {
IExpectedSchemaNode rootNode = varToNode.get(recordVariable);
- Map<String, FunctionCallInformation> sourceInformation = new HashMap<>();
- typeBuilder.reset(sourceInformation);
+
+ ExpectedSchemaNodeToIATypeTranslatorVisitor typeBuilder =
+ new ExpectedSchemaNodeToIATypeTranslatorVisitor(sourceInformationMap);
ARecordType recordType = (ARecordType) rootNode.accept(typeBuilder, null);
- return new DataProjectionInfo(recordType, sourceInformation);
+
+ return new DataProjectionFiltrationInfo(recordType, sourceInformationMap, filterPaths, filterExpression);
}
- public boolean setSchemaFromExpression(AbstractFunctionCallExpression expr, LogicalVariable producedVar) {
+ public boolean setSchemaFromExpression(AbstractFunctionCallExpression expr, LogicalVariable producedVar,
+ IVariableTypeEnvironment typeEnv) throws AlgebricksException {
//Parent always nested
- AbstractComplexExpectedSchemaNode parent = (AbstractComplexExpectedSchemaNode) buildNestedNode(expr);
+ AbstractComplexExpectedSchemaNode parent = (AbstractComplexExpectedSchemaNode) buildNestedNode(expr, typeEnv);
if (parent != null) {
IExpectedSchemaNode leaf =
new AnyExpectedSchemaNode(parent, expr.getSourceLocation(), expr.getFunctionIdentifier().getName());
- addChild(expr, parent, leaf);
+ addChild(expr, typeEnv, parent, leaf);
+
+ //Associate expression to node
+ exprToNode.put(expr, leaf);
if (producedVar != null) {
//Register the node if a variable is produced
varToNode.put(producedVar, leaf);
@@ -86,7 +104,7 @@
return parent != null;
}
- public void registerDataset(LogicalVariable recordVar, RootExpectedSchemaNode rootNode) {
+ public void registerRoot(LogicalVariable recordVar, RootExpectedSchemaNode rootNode) {
varToNode.put(recordVar, rootNode);
}
@@ -111,10 +129,19 @@
return !varToNode.isEmpty();
}
- private IExpectedSchemaNode buildNestedNode(ILogicalExpression expr) {
+ IExpectedSchemaNode getNode(LogicalVariable variable) {
+ return varToNode.get(variable);
+ }
+
+ IExpectedSchemaNode getNode(AbstractFunctionCallExpression expr) {
+ return exprToNode.get(expr);
+ }
+
+ private IExpectedSchemaNode buildNestedNode(ILogicalExpression expr, IVariableTypeEnvironment typeEnv)
+ throws AlgebricksException {
//The current node expression
AbstractFunctionCallExpression myExpr = (AbstractFunctionCallExpression) expr;
- if (!SUPPORTED_FUNCTIONS.contains(myExpr.getFunctionIdentifier())) {
+ if (!SUPPORTED_FUNCTIONS.contains(myExpr.getFunctionIdentifier()) || noArgsOrFirstArgIsConstant(myExpr)) {
//Return null if the function is not supported.
return null;
}
@@ -128,7 +155,8 @@
}
//Recursively create the parent nodes. Parent is always a nested node
- AbstractComplexExpectedSchemaNode newParent = (AbstractComplexExpectedSchemaNode) buildNestedNode(parentExpr);
+ AbstractComplexExpectedSchemaNode newParent =
+ (AbstractComplexExpectedSchemaNode) buildNestedNode(parentExpr, typeEnv);
//newParent could be null if the expression is not supported
if (newParent != null) {
//Parent expression must be a function call (as parent is a nested node)
@@ -136,18 +164,23 @@
//Get 'myType' as we will create the child type of the newParent
ExpectedSchemaNodeType myType = getExpectedNestedNodeType(myExpr);
/*
- * Create 'myNode'. It is a nested node because the function is either getField() or supported array
+ * Create 'myNode'. It is a nested node because the function is either getField() or a supported array
* function
*/
AbstractComplexExpectedSchemaNode myNode = AbstractComplexExpectedSchemaNode.createNestedNode(myType,
newParent, myExpr.getSourceLocation(), myExpr.getFunctionIdentifier().getName());
//Add myNode to the parent
- addChild(parentFuncExpr, newParent, myNode);
+ addChild(parentFuncExpr, typeEnv, newParent, myNode);
return myNode;
}
return null;
}
+ private boolean noArgsOrFirstArgIsConstant(AbstractFunctionCallExpression myExpr) {
+ List<Mutable<ILogicalExpression>> args = myExpr.getArguments();
+ return args.isEmpty() || args.get(0).getValue().getExpressionTag() == LogicalExpressionTag.CONSTANT;
+ }
+
private IExpectedSchemaNode changeNodeForVariable(LogicalVariable sourceVar,
AbstractFunctionCallExpression myExpr) {
//Get the associated node with the sourceVar (if any)
@@ -166,11 +199,11 @@
return newNode;
}
- private void addChild(AbstractFunctionCallExpression parentExpr, AbstractComplexExpectedSchemaNode parent,
- IExpectedSchemaNode child) {
+ private void addChild(AbstractFunctionCallExpression parentExpr, IVariableTypeEnvironment typeEnv,
+ AbstractComplexExpectedSchemaNode parent, IExpectedSchemaNode child) throws AlgebricksException {
switch (parent.getType()) {
case OBJECT:
- handleObject(parentExpr, parent, child);
+ handleObject(parentExpr, typeEnv, parent, child);
break;
case ARRAY:
handleArray(parent, child);
@@ -184,10 +217,18 @@
}
}
- private void handleObject(AbstractFunctionCallExpression parentExpr, AbstractComplexExpectedSchemaNode parent,
- IExpectedSchemaNode child) {
- ObjectExpectedSchemaNode objectNode = (ObjectExpectedSchemaNode) parent;
- objectNode.addChild(ConstantExpressionUtil.getStringArgument(parentExpr, 1), child);
+ private void handleObject(AbstractFunctionCallExpression parentExpr, IVariableTypeEnvironment typeEnv,
+ AbstractComplexExpectedSchemaNode parent, IExpectedSchemaNode child) throws AlgebricksException {
+ if (BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(parentExpr.getFunctionIdentifier())) {
+ ObjectExpectedSchemaNode objectNode = (ObjectExpectedSchemaNode) parent;
+ objectNode.addChild(ConstantExpressionUtil.getStringArgument(parentExpr, 1), child);
+ } else {
+ //FIELD_ACCESS_BY_INDEX
+ ARecordType recordType = (ARecordType) typeEnv.getType(parentExpr.getArguments().get(0).getValue());
+ ObjectExpectedSchemaNode objectNode = (ObjectExpectedSchemaNode) parent;
+ int fieldIdx = ConstantExpressionUtil.getIntArgument(parentExpr, 1);
+ objectNode.addChild(recordType.getFieldNames()[fieldIdx], child);
+ }
}
private void handleArray(AbstractComplexExpectedSchemaNode parent, IExpectedSchemaNode child) {
@@ -196,15 +237,15 @@
}
private void handleUnion(AbstractFunctionCallExpression parentExpr, AbstractComplexExpectedSchemaNode parent,
- IExpectedSchemaNode child) {
+ IExpectedSchemaNode child) throws AlgebricksException {
UnionExpectedSchemaNode unionNode = (UnionExpectedSchemaNode) parent;
ExpectedSchemaNodeType parentType = getExpectedNestedNodeType(parentExpr);
- addChild(parentExpr, unionNode.getChild(parentType), child);
+ addChild(parentExpr, null, unionNode.getChild(parentType), child);
}
private static ExpectedSchemaNodeType getExpectedNestedNodeType(AbstractFunctionCallExpression funcExpr) {
FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
- if (BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fid)) {
+ if (BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fid) || BuiltinFunctions.FIELD_ACCESS_BY_INDEX.equals(fid)) {
return ExpectedSchemaNodeType.OBJECT;
} else if (ARRAY_FUNCTIONS.contains(fid)) {
return ExpectedSchemaNodeType.ARRAY;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaNodeToIATypeTranslatorVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaNodeToIATypeTranslatorVisitor.java
index c746994..32e4b18 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaNodeToIATypeTranslatorVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpectedSchemaNodeToIATypeTranslatorVisitor.java
@@ -37,29 +37,30 @@
import org.apache.asterix.optimizer.rules.pushdown.schema.ObjectExpectedSchemaNode;
import org.apache.asterix.optimizer.rules.pushdown.schema.RootExpectedSchemaNode;
import org.apache.asterix.optimizer.rules.pushdown.schema.UnionExpectedSchemaNode;
-import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.asterix.runtime.projection.ProjectionFiltrationWarningFactoryProvider;
/**
* This visitor translates the {@link IExpectedSchemaNode} to {@link IAType} record.
* The {@link IAType#getTypeName()} is used to map each {@link IAType} to its {@link FunctionCallInformation}
*/
-class ExpectedSchemaNodeToIATypeTranslatorVisitor implements IExpectedSchemaNodeVisitor<IAType, String> {
+public class ExpectedSchemaNodeToIATypeTranslatorVisitor implements IExpectedSchemaNodeVisitor<IAType, String> {
//Map typeName to source information
- private Map<String, FunctionCallInformation> sourceInformationMap;
+ private final Map<String, FunctionCallInformation> sourceInformationMap;
//To give a unique name for each type
private int counter;
- public void reset(Map<String, FunctionCallInformation> sourceInformationMap) {
+ public ExpectedSchemaNodeToIATypeTranslatorVisitor(Map<String, FunctionCallInformation> sourceInformationMap) {
this.sourceInformationMap = sourceInformationMap;
}
@Override
public IAType visit(RootExpectedSchemaNode node, String arg) {
if (node.isAllFields()) {
- return DataProjectionInfo.ALL_FIELDS_TYPE;
+ return DataProjectionFiltrationInfo.ALL_FIELDS_TYPE;
} else if (node.isEmpty()) {
- return DataProjectionInfo.EMPTY_TYPE;
+ return DataProjectionFiltrationInfo.EMPTY_TYPE;
}
return createRecordType(node, String.valueOf(counter++));
}
@@ -109,6 +110,7 @@
}
private FunctionCallInformation createFunctionCallInformation(IExpectedSchemaNode node) {
- return new FunctionCallInformation(node.getFunctionName(), node.getSourceLocation());
+ return new FunctionCallInformation(node.getFunctionName(), node.getSourceLocation(),
+ ProjectionFiltrationWarningFactoryProvider.TYPE_MISMATCH_FACTORY);
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpressionValueAccessPushdownVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpressionValueAccessPushdownVisitor.java
index a8dfe1e..8096b04 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpressionValueAccessPushdownVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpressionValueAccessPushdownVisitor.java
@@ -29,21 +29,23 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
class ExpressionValueAccessPushdownVisitor implements ILogicalExpressionReferenceTransform {
- //Set of supported type-check functions
- static final Set<FunctionIdentifier> TYPE_CHECK_FUNCTIONS = createSupportedTypeCheckFunctions();
+ //Set of allowed functions that can request a type in its entirety
+ static final Set<FunctionIdentifier> ALLOWED_FUNCTIONS = createAllowedFunctions();
//Set of supported array functions
static final Set<FunctionIdentifier> ARRAY_FUNCTIONS = createSupportedArrayFunctions();
- //Set of supported functions that we can pushdown
+ //Set of supported functions that we can push down
static final Set<FunctionIdentifier> SUPPORTED_FUNCTIONS = createSupportedFunctions();
private final ExpectedSchemaBuilder builder;
private List<LogicalVariable> producedVariables;
+ private IVariableTypeEnvironment typeEnv;
private int producedVariableIndex;
public ExpressionValueAccessPushdownVisitor(ExpectedSchemaBuilder builder) {
@@ -51,8 +53,9 @@
end();
}
- public void init(List<LogicalVariable> producedVariables) {
+ public void init(List<LogicalVariable> producedVariables, IVariableTypeEnvironment typeEnv) {
this.producedVariables = producedVariables;
+ this.typeEnv = typeEnv;
producedVariableIndex = 0;
}
@@ -62,12 +65,13 @@
//This for ensuring that the produced variables (if any) should be set
throw new IllegalStateException("init must be called first");
}
- pushValueAccessExpression(expression, getNextProducedVariable());
+ pushValueAccessExpression(expression, getNextProducedVariable(), typeEnv);
return false;
}
public void end() {
producedVariables = null;
+ typeEnv = null;
producedVariableIndex = -1;
}
@@ -80,7 +84,8 @@
/**
* Pushdown field access expressions and array access expressions down
*/
- private void pushValueAccessExpression(Mutable<ILogicalExpression> exprRef, LogicalVariable producedVar) {
+ private void pushValueAccessExpression(Mutable<ILogicalExpression> exprRef, LogicalVariable producedVar,
+ IVariableTypeEnvironment typeEnv) throws AlgebricksException {
final ILogicalExpression expr = exprRef.getValue();
if (skipPushdown(expr)) {
return;
@@ -88,7 +93,7 @@
final AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
- if (isSuccessfullyPushedDown(funcExpr, producedVar)) {
+ if (isSuccessfullyPushedDown(funcExpr, producedVar, typeEnv)) {
//We successfully pushed down the value access function
return;
}
@@ -138,23 +143,24 @@
*/
private boolean isTypeCheckOnVariable(ILogicalExpression expression) {
AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
- return TYPE_CHECK_FUNCTIONS.contains(funcExpr.getFunctionIdentifier())
+ return ALLOWED_FUNCTIONS.contains(funcExpr.getFunctionIdentifier())
&& funcExpr.getArguments().get(0).getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE;
}
- private void pushValueAccessExpressionArg(List<Mutable<ILogicalExpression>> exprList) {
+ private void pushValueAccessExpressionArg(List<Mutable<ILogicalExpression>> exprList) throws AlgebricksException {
for (Mutable<ILogicalExpression> exprRef : exprList) {
/*
* We need to set the produced variable as null here as the produced variable will not correspond to the
* nested expression.
*/
- pushValueAccessExpression(exprRef, null);
+ pushValueAccessExpression(exprRef, null, typeEnv);
}
}
- private boolean isSuccessfullyPushedDown(AbstractFunctionCallExpression funcExpr, LogicalVariable producedVar) {
+ private boolean isSuccessfullyPushedDown(AbstractFunctionCallExpression funcExpr, LogicalVariable producedVar,
+ IVariableTypeEnvironment typeEnv) throws AlgebricksException {
return SUPPORTED_FUNCTIONS.contains(funcExpr.getFunctionIdentifier())
- && builder.setSchemaFromExpression(funcExpr, producedVar);
+ && builder.setSchemaFromExpression(funcExpr, producedVar, typeEnv);
}
private void unregisterVariableIfNeeded(LogicalVariable variable) {
@@ -169,15 +175,17 @@
private static Set<FunctionIdentifier> createSupportedFunctions() {
Set<FunctionIdentifier> supportedFunctions = new HashSet<>();
- //For objects, only field-access-by-name is supported
supportedFunctions.add(BuiltinFunctions.FIELD_ACCESS_BY_NAME);
+ supportedFunctions.add(BuiltinFunctions.FIELD_ACCESS_BY_INDEX);
supportedFunctions.addAll(ARRAY_FUNCTIONS);
return supportedFunctions;
}
- private static Set<FunctionIdentifier> createSupportedTypeCheckFunctions() {
+ private static Set<FunctionIdentifier> createAllowedFunctions() {
return Set.of(BuiltinFunctions.IS_ARRAY, BuiltinFunctions.IS_OBJECT, BuiltinFunctions.IS_ATOMIC,
BuiltinFunctions.IS_NUMBER, BuiltinFunctions.IS_BOOLEAN, BuiltinFunctions.IS_STRING,
- AlgebricksBuiltinFunctions.IS_MISSING, AlgebricksBuiltinFunctions.IS_NULL, BuiltinFunctions.IS_UNKNOWN);
+ AlgebricksBuiltinFunctions.IS_MISSING, AlgebricksBuiltinFunctions.IS_NULL, BuiltinFunctions.IS_UNKNOWN,
+ BuiltinFunctions.LT, BuiltinFunctions.LE, BuiltinFunctions.EQ, BuiltinFunctions.GT, BuiltinFunctions.GE,
+ BuiltinFunctions.SCALAR_SQL_COUNT);
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpressionValueFilterPushdown.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpressionValueFilterPushdown.java
new file mode 100644
index 0000000..8ca2372
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/ExpressionValueFilterPushdown.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules.pushdown;
+
+import static org.apache.asterix.metadata.utils.ColumnFilterBuilder.COMPARE_FUNCTIONS;
+import static org.apache.asterix.metadata.utils.ColumnFilterBuilder.PUSHABLE_FUNCTIONS;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.metadata.declared.DatasetDataSource;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.optimizer.rules.pushdown.schema.AnyExpectedSchemaNode;
+import org.apache.asterix.optimizer.rules.pushdown.schema.ColumnFilterPathBuilderVisitor;
+import org.apache.asterix.optimizer.rules.pushdown.schema.ExpectedSchemaNodeType;
+import org.apache.asterix.optimizer.rules.pushdown.schema.IExpectedSchemaNode;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.asterix.runtime.projection.ProjectionFiltrationWarningFactoryProvider;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+/**
+ * Pushdown {@link SelectOperator} condition to the dataset to allow filtering mega leaf nodes.
+ * This is currently only allowed for {@link DatasetConfig.DatasetFormat#COLUMN}
+ * TODO allow partial filter with AND (e.g., lowercase(stringVal) == "some_text" AND intVal > 10 --push--> intVal > 10 )
+ * TODO Filter could prevent REPLICATE (i.e., we can scan a dataset twice due to the fact one scan is filtered and
+ * TODO the other is not) or (both have different filters)
+ * TODO part of this class could potentially be used for external data dynamic prefixes
+ */
+class ExpressionValueFilterPushdown {
+ private final ExpectedSchemaBuilder builder;
+ private final ColumnFilterPathBuilderVisitor pathBuilder;
+ private final Map<AbstractScanOperator, Map<ILogicalExpression, ARecordType>> datasetFilterPaths;
+ private final Map<AbstractScanOperator, ILogicalExpression> datasetFilterExpression;
+ private final Map<AbstractScanOperator, Map<String, FunctionCallInformation>> scanSourceInformationMaps;
+ private final Set<AbstractScanOperator> registeredScans;
+ private final boolean columnFilterEnabled;
+
+ ExpressionValueFilterPushdown(ExpectedSchemaBuilder builder, boolean columnFilterEnabled) {
+ this.builder = builder;
+ pathBuilder = new ColumnFilterPathBuilderVisitor();
+ datasetFilterPaths = new HashMap<>();
+ datasetFilterExpression = new HashMap<>();
+ scanSourceInformationMaps = new HashMap<>();
+ registeredScans = new HashSet<>();
+ this.columnFilterEnabled = columnFilterEnabled;
+ }
+
+ public void registerDataset(AbstractScanOperator op, DatasetDataSource source) {
+ if (!columnFilterEnabled) {
+ return;
+ }
+
+ Dataset dataset = source.getDataset();
+ if (dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL
+ && dataset.getDatasetFormatInfo().getFormat() == DatasetConfig.DatasetFormat.COLUMN) {
+ registeredScans.add(op);
+ }
+ }
+
+ /**
+ * Try to push the condition down to dataset scan/access
+ *
+ * @param selectOp the select operator
+ */
+ public void addFilterExpression(SelectOperator selectOp, AbstractScanOperator scanOp) {
+ if (datasetFilterPaths.containsKey(scanOp)) {
+ // Most bottom SELECT was applied, other selects should be ignored
+ return;
+ }
+ Map<ILogicalExpression, ARecordType> filterPaths = new HashMap<>();
+ Map<String, FunctionCallInformation> sourceInformationMap = new HashMap<>();
+ Mutable<ILogicalExpression> conditionRef = selectOp.getCondition();
+ if (addPaths(conditionRef, filterPaths, sourceInformationMap)) {
+ datasetFilterPaths.put(scanOp, filterPaths);
+ datasetFilterExpression.put(scanOp, conditionRef.getValue());
+ scanSourceInformationMaps.put(scanOp, sourceInformationMap);
+ }
+ }
+
+ public Map<ILogicalExpression, ARecordType> getFilterPaths(AbstractScanOperator scanOp) {
+ return datasetFilterPaths.getOrDefault(scanOp, Collections.emptyMap());
+ }
+
+ public ILogicalExpression getFilterExpression(AbstractScanOperator scanOp) {
+ return datasetFilterExpression.get(scanOp);
+ }
+
+ public Map<String, FunctionCallInformation> getSourceInformationMap(AbstractScanOperator scanOp) {
+ return scanSourceInformationMaps.getOrDefault(scanOp, new HashMap<>());
+ }
+
+ private boolean addPaths(Mutable<ILogicalExpression> exprRef, Map<ILogicalExpression, ARecordType> filterPaths,
+ Map<String, FunctionCallInformation> sourceInformationMap) {
+ ILogicalExpression expr = exprRef.getValue();
+ if (!isFunctionExpression(expr)) {
+ return false;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ FunctionIdentifier fid = ((AbstractFunctionCallExpression) expr).getFunctionIdentifier();
+
+ if (!PUSHABLE_FUNCTIONS.contains(fid)) {
+ return false;
+ }
+
+ if (COMPARE_FUNCTIONS.contains(fid)) {
+ return addPaths(funcExpr, filterPaths, sourceInformationMap);
+ }
+ //AND/OR descend to the expression tree
+ return addPathsForArgs(funcExpr, filterPaths, sourceInformationMap);
+ }
+
+ private boolean addPaths(AbstractFunctionCallExpression funcExpr, Map<ILogicalExpression, ARecordType> filterPaths,
+ Map<String, FunctionCallInformation> sourceInformationMap) {
+ List<Mutable<ILogicalExpression>> args = funcExpr.getArguments();
+
+ ILogicalExpression left = args.get(0).getValue();
+ ILogicalExpression right = args.get(1).getValue();
+
+ if (isConstantExpression(left)) {
+ return addPaths(funcExpr, right, left, filterPaths, sourceInformationMap, true);
+ } else if (isConstantExpression(right)) {
+ return addPaths(funcExpr, left, right, filterPaths, sourceInformationMap, false);
+ }
+ // No constants, return false
+ return false;
+ }
+
+ private boolean addPaths(AbstractFunctionCallExpression funcExpr, ILogicalExpression columnExpr,
+ ILogicalExpression constExpr, Map<ILogicalExpression, ARecordType> filterPaths,
+ Map<String, FunctionCallInformation> sourceInformationMap, boolean leftConstant) {
+ IExpectedSchemaNode node;
+ if (isFunctionExpression(columnExpr)) {
+ node = builder.getNode((AbstractFunctionCallExpression) columnExpr);
+ } else {
+ //Variable
+ node = builder.getNode(((VariableReferenceExpression) columnExpr).getVariableReference());
+ }
+
+ if (node == null || node.getType() != ExpectedSchemaNodeType.ANY) {
+ // Expression cannot be pushed (e.g., $$r.getField("x") + 1) or had been accessed as a nested value
+ // Bail out
+ return false;
+ }
+
+ AnyExpectedSchemaNode leafNode = (AnyExpectedSchemaNode) node;
+ IAObject constantValue = ((AsterixConstantValue) ((ConstantExpression) constExpr).getValue()).getObject();
+
+ String functionName = funcExpr.getFunctionIdentifier().getName();
+ SourceLocation sourceLocation = funcExpr.getSourceLocation();
+ FunctionCallInformation functionCallInfo = new FunctionCallInformation(functionName, sourceLocation,
+ ProjectionFiltrationWarningFactoryProvider.getIncomparableTypesFactory(leftConstant));
+
+ ARecordType path = pathBuilder.buildPath(leafNode, constantValue, sourceInformationMap, functionCallInfo);
+ filterPaths.put(columnExpr, path);
+ return true;
+ }
+
+ private boolean addPathsForArgs(AbstractFunctionCallExpression funcExpr,
+ Map<ILogicalExpression, ARecordType> filterPaths,
+ Map<String, FunctionCallInformation> sourceInformationMap) {
+ List<Mutable<ILogicalExpression>> args = funcExpr.getArguments();
+ boolean add = true;
+ for (int i = 0; add && i < args.size(); i++) {
+ add = addPaths(args.get(i), filterPaths, sourceInformationMap);
+ }
+ return add;
+ }
+
+ private static boolean isFunctionExpression(ILogicalExpression expr) {
+ return expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL;
+ }
+
+ private static boolean isConstantExpression(ILogicalExpression expr) {
+ return expr.getExpressionTag() == LogicalExpressionTag.CONSTANT;
+ }
+
+ public boolean allowsPushdown(AbstractScanOperator lastSeenScan) {
+ return columnFilterEnabled && lastSeenScan != null && registeredScans.contains(lastSeenScan);
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/OperatorValueAccessPushdownVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/OperatorValueAccessPushdownVisitor.java
index 6739384..8fe8ce7 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/OperatorValueAccessPushdownVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/OperatorValueAccessPushdownVisitor.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.optimizer.rules.pushdown;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -25,15 +26,20 @@
import java.util.Set;
import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.common.config.DatasetConfig.DatasetFormat;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.metadata.declared.DataSource;
+import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.DatasetDataSource;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.asterix.optimizer.rules.pushdown.schema.RootExpectedSchemaNode;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -44,7 +50,9 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
@@ -74,6 +82,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -87,28 +96,60 @@
* This visitor visits the entire plan and tries to build the information of the required values from all dataset
*/
public class OperatorValueAccessPushdownVisitor implements ILogicalOperatorVisitor<Void, Void> {
+ private static final List<LogicalVariable> EMPTY_VARIABLES = Collections.emptyList();
private final IOptimizationContext context;
//Requested schema builder. It is only expected schema not a definite one
private final ExpectedSchemaBuilder builder;
//To visit every expression in each operator
private final ExpressionValueAccessPushdownVisitor pushdownVisitor;
+ private final ExpressionValueFilterPushdown filterPushdown;
//Datasets that allow pushdowns
- private final Map<LogicalVariable, DataSourceScanOperator> registeredDatasets;
+ private final Map<LogicalVariable, AbstractScanOperator> registeredDatasets;
+ //Datasets that allow pushdowns and has meta
+ private final Map<LogicalVariable, AbstractScanOperator> registeredMetas;
//visitedOperators so we do not visit the same operator twice (in case of REPLICATE)
private final Set<ILogicalOperator> visitedOperators;
+ //Last scan operator seen
+ private AbstractScanOperator lastSeenScan;
public OperatorValueAccessPushdownVisitor(IOptimizationContext context) {
this.context = context;
builder = new ExpectedSchemaBuilder();
registeredDatasets = new HashMap<>();
+ registeredMetas = new HashMap<>();
pushdownVisitor = new ExpressionValueAccessPushdownVisitor(builder);
+ filterPushdown = new ExpressionValueFilterPushdown(builder,
+ context.getPhysicalOptimizationConfig().isColumnFilterEnabled());
visitedOperators = new HashSet<>();
}
public void finish() {
- for (Map.Entry<LogicalVariable, DataSourceScanOperator> scan : registeredDatasets.entrySet()) {
- scan.getValue().setProjectionInfo(builder.createProjectionInfo(scan.getKey()));
+ for (Map.Entry<LogicalVariable, AbstractScanOperator> entry : registeredDatasets.entrySet()) {
+ AbstractScanOperator scanOp = entry.getValue();
+ Map<ILogicalExpression, ARecordType> filterPaths = filterPushdown.getFilterPaths(scanOp);
+ ILogicalExpression filterExpression = filterPushdown.getFilterExpression(scanOp);
+ Map<String, FunctionCallInformation> sourceInformationMap = filterPushdown.getSourceInformationMap(scanOp);
+ if (scanOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ DataSourceScanOperator scan = (DataSourceScanOperator) scanOp;
+ scan.setDatasetProjectionInfo(builder.createProjectionInfo(entry.getKey(), filterPaths,
+ filterExpression, sourceInformationMap));
+ } else {
+ UnnestMapOperator unnest = (UnnestMapOperator) scanOp;
+ unnest.setDatasetProjectionInfo(builder.createProjectionInfo(entry.getKey(), filterPaths,
+ filterExpression, sourceInformationMap));
+ }
+ }
+
+ for (Map.Entry<LogicalVariable, AbstractScanOperator> entry : registeredMetas.entrySet()) {
+ AbstractScanOperator abstractScan = entry.getValue();
+ if (abstractScan.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ DataSourceScanOperator scan = (DataSourceScanOperator) abstractScan;
+ scan.setMetaProjectionInfo(builder.createProjectionInfo(entry.getKey()));
+ } else {
+ UnnestMapOperator unnest = (UnnestMapOperator) abstractScan;
+ unnest.setMetaProjectionInfo(builder.createProjectionInfo(entry.getKey()));
+ }
}
}
@@ -126,11 +167,18 @@
for (Mutable<ILogicalOperator> child : op.getInputs()) {
child.getValue().accept(this, null);
}
+ IVariableTypeEnvironment typeEnv = op.computeOutputTypeEnvironment(context);
visitedOperators.add(op);
//Initiate the pushdown visitor
- pushdownVisitor.init(producedVariables);
+ pushdownVisitor.init(producedVariables, typeEnv);
//pushdown any expression the operator has
op.acceptExpressionTransform(pushdownVisitor);
+
+ if (filterPushdown.allowsPushdown(lastSeenScan) && op.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ //Push filters down
+ filterPushdown.addFilterExpression((SelectOperator) op, lastSeenScan);
+ }
+
pushdownVisitor.end();
}
@@ -143,10 +191,8 @@
@Override
public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
visitInputs(op);
- if (op.getVariables().isEmpty()) {
- //If the variables are empty and the next operator is DataSourceScanOperator, then set empty record
- setEmptyRecord(op.getInputs().get(0).getValue());
- }
+ //Set as empty records for data-scan or unnest-map if certain variables are projected out
+ setEmptyRecord(op.getInputs().get(0).getValue(), op.getVariables());
return null;
}
@@ -156,20 +202,21 @@
*/
@Override
public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+ DatasetDataSource datasetDataSource = getDatasetDataSourceIfApplicable((DataSource) op.getDataSource());
+ registerDatasetIfApplicable(datasetDataSource, op);
visitInputs(op);
- DatasetDataSource datasetDataSource = getDatasetDataSourceIfApplicable(op);
- if (datasetDataSource != null) {
- LogicalVariable recordVar = datasetDataSource.getDataRecordVariable(op.getVariables());
- if (!builder.isVariableRegistered(recordVar)) {
- /*
- * This is the first time we see the dataset, and we know we might only need part of the record.
- * Register the dataset to prepare for value access expression pushdowns.
- * Initially, we will request the entire record.
- */
- builder.registerDataset(recordVar, RootExpectedSchemaNode.ALL_FIELDS_ROOT_NODE);
- registeredDatasets.put(recordVar, op);
- }
- }
+ return null;
+ }
+
+ /**
+ * From the {@link UnnestMapOperator}, we need to register the payload variable (record variable) to check
+ * which expression in the plan is using it.
+ */
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+ visitInputs(op);
+ DatasetDataSource datasetDataSource = getDatasetDataSourceIfApplicable(getDataSourceFromUnnestMapOperator(op));
+ registerDatasetIfApplicable(datasetDataSource, op);
return null;
}
@@ -182,7 +229,7 @@
* It is local aggregate and has agg-sql-count function with a constant argument. Set empty record if the
* input operator is DataSourceScanOperator
*/
- setEmptyRecord(op.getInputs().get(0).getValue());
+ setEmptyRecord(op.getInputs().get(0).getValue(), EMPTY_VARIABLES);
}
return null;
}
@@ -195,11 +242,10 @@
/**
* The role of this method is:
- * 1- Check whether the dataset is an external dataset and allows value access pushdowns
+ * 1- Check whether the datasource allows value access pushdowns
* 2- return the actual DatasetDataSource
*/
- private DatasetDataSource getDatasetDataSourceIfApplicable(DataSourceScanOperator scan) throws AlgebricksException {
- DataSource dataSource = (DataSource) scan.getDataSource();
+ private DatasetDataSource getDatasetDataSourceIfApplicable(DataSource dataSource) throws AlgebricksException {
if (dataSource == null) {
return null;
}
@@ -210,9 +256,11 @@
Dataset dataset = mp.findDataset(dataverse, datasetName);
//Only external dataset can have pushed down expressions
- if (dataset == null || dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL
- || dataset.getDatasetType() == DatasetConfig.DatasetType.EXTERNAL && !ExternalDataUtils
- .supportsPushdown(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties())) {
+ if (dataset.getDatasetType() == DatasetConfig.DatasetType.EXTERNAL
+ && !ExternalDataUtils
+ .supportsPushdown(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties())
+ || dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL
+ && dataset.getDatasetFormatInfo().getFormat() == DatasetFormat.ROW) {
return null;
}
@@ -220,24 +268,116 @@
}
/**
- * If the inputOp is a {@link DataSourceScanOperator}, then set the projected value needed as empty record
+ * Find datasource from {@link UnnestMapOperator}
*
- * @param inputOp an operator that is potentially a {@link DataSourceScanOperator}
+ * @param unnest unnest map operator
+ * @return datasource
+ */
+ private DataSource getDataSourceFromUnnestMapOperator(UnnestMapOperator unnest) throws AlgebricksException {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) unnest.getExpressionRef().getValue();
+ String dataverse = ConstantExpressionUtil.getStringArgument(funcExpr, 2);
+ String dataset = ConstantExpressionUtil.getStringArgument(funcExpr, 3);
+ if (!ConstantExpressionUtil.getStringArgument(funcExpr, 0).equals(dataset)) {
+ return null;
+ }
+
+ DataSourceId dsid = new DataSourceId(DataverseName.createBuiltinDataverseName(dataverse), dataset);
+ MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+ return metadataProvider.findDataSource(dsid);
+ }
+
+ private void registerDatasetIfApplicable(DatasetDataSource datasetDataSource, AbstractScanOperator op) {
+ if (datasetDataSource != null) {
+ LogicalVariable recordVar = datasetDataSource.getDataRecordVariable(op.getVariables());
+ if (!builder.isVariableRegistered(recordVar)) {
+ /*
+ * This is the first time we see the dataset, and we know we might only need part of the record.
+ * Register the dataset to prepare for value access expression pushdowns.
+ * Initially, we will request the entire record.
+ */
+ builder.registerRoot(recordVar, RootExpectedSchemaNode.ALL_FIELDS_ROOT_NODE);
+ filterPushdown.registerDataset(op, datasetDataSource);
+ registeredDatasets.put(recordVar, op);
+
+ if (datasetDataSource.hasMeta()) {
+ /*
+ * The dataset has meta. Register the meta root variable as another root for the dataset and add
+ * it the metaVar to the registered metas
+ */
+ LogicalVariable metaVar = datasetDataSource.getMetaVariable(op.getVariables());
+ builder.registerRoot(metaVar, RootExpectedSchemaNode.ALL_FIELDS_ROOT_NODE);
+ registeredMetas.put(metaVar, op);
+ }
+ }
+ lastSeenScan = op;
+ }
+ }
+
+ /**
+ * If the inputOp is a {@link DataSourceScanOperator} or {@link UnnestMapOperator}, then set the projected value
+ * needed as empty record if any variable originated from either operators are not in {@code retainedVariables}
+ *
+ * @param inputOp an operator that is potentially a {@link DataSourceScanOperator} or a {@link
+ * UnnestMapOperator}
+ * @param retainedVariables variables that should be retained
* @see #visitAggregateOperator(AggregateOperator, Void)
* @see #visitProjectOperator(ProjectOperator, Void)
*/
- private void setEmptyRecord(ILogicalOperator inputOp) throws AlgebricksException {
+ private void setEmptyRecord(ILogicalOperator inputOp, List<LogicalVariable> retainedVariables)
+ throws AlgebricksException {
+ LogicalOperatorTag tag = inputOp.getOperatorTag();
+ if (tag != LogicalOperatorTag.DATASOURCESCAN && tag != LogicalOperatorTag.UNNEST_MAP) {
+ return;
+ }
+
+ DataSource dataSource;
+ List<LogicalVariable> variables;
+ Mutable<ILogicalExpression> selectCondition;
if (inputOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
DataSourceScanOperator scan = (DataSourceScanOperator) inputOp;
- DatasetDataSource datasetDataSource = getDatasetDataSourceIfApplicable(scan);
- if (datasetDataSource != null) {
- //We know that we only need the count of objects. So return empty objects only
- LogicalVariable recordVar = datasetDataSource.getDataRecordVariable(scan.getVariables());
- /*
- * Set the root node as EMPTY_ROOT_NODE (i.e., no fields will be read from disk). We register the
- * dataset with EMPTY_ROOT_NODE so that we skip pushdowns on empty node.
- */
- builder.registerDataset(recordVar, RootExpectedSchemaNode.EMPTY_ROOT_NODE);
+ dataSource = (DataSource) scan.getDataSource();
+ variables = scan.getVariables();
+ selectCondition = scan.getSelectCondition();
+ } else {
+ UnnestMapOperator unnest = (UnnestMapOperator) inputOp;
+ dataSource = getDataSourceFromUnnestMapOperator(unnest);
+ variables = unnest.getVariables();
+ selectCondition = unnest.getSelectCondition();
+ }
+
+ DatasetDataSource datasetDataSource = getDatasetDataSourceIfApplicable(dataSource);
+
+ if (datasetDataSource == null) {
+ //Does not support pushdown
+ return;
+ }
+
+ Set<LogicalVariable> selectConditionVariables = new HashSet<>();
+ if (selectCondition != null) {
+ //Get the used variables for a select condition
+ selectCondition.getValue().getUsedVariables(selectConditionVariables);
+ }
+
+ //We know that we only need the count of objects. So return empty objects only
+ LogicalVariable recordVar = datasetDataSource.getDataRecordVariable(variables);
+
+ /*
+ * If the recordVar is not retained by an upper operator and not used by a select condition, then return empty
+ * record instead of the entire record.
+ */
+ if (!retainedVariables.contains(recordVar) && !selectConditionVariables.contains(recordVar)) {
+ /*
+ * Set the root node as EMPTY_ROOT_NODE (i.e., no fields will be read from disk). We register the
+ * dataset with EMPTY_ROOT_NODE so that we skip pushdowns on empty node.
+ */
+ builder.registerRoot(recordVar, RootExpectedSchemaNode.EMPTY_ROOT_NODE);
+ }
+
+ if (datasetDataSource.hasMeta()) {
+ //Do the same for meta
+ LogicalVariable metaVar = datasetDataSource.getMetaVariable(variables);
+ if (!retainedVariables.contains(metaVar)) {
+ builder.registerRoot(metaVar, RootExpectedSchemaNode.EMPTY_ROOT_NODE);
}
}
}
@@ -362,6 +502,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ visitInputs(op);
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
visitInputs(op);
return null;
@@ -398,12 +544,6 @@
}
@Override
- public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
- visitInputs(op);
- return null;
- }
-
- @Override
public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) throws AlgebricksException {
visitInputs(op);
return null;
@@ -474,4 +614,4 @@
private void visitInputs(ILogicalOperator op) throws AlgebricksException {
visitInputs(op, null);
}
-}
\ No newline at end of file
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ColumnFilterPathBuilderVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ColumnFilterPathBuilderVisitor.java
new file mode 100644
index 0000000..3f83834
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ColumnFilterPathBuilderVisitor.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules.pushdown.schema;
+
+import java.util.Map;
+
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.asterix.runtime.projection.ProjectionFiltrationWarningFactoryProvider;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class ColumnFilterPathBuilderVisitor implements IExpectedSchemaNodeVisitor<IAType, IExpectedSchemaNode> {
+
+ private IAType type;
+ private Map<String, FunctionCallInformation> sourceInformationMap;
+ private int counter = 0;
+
+ public ARecordType buildPath(AnyExpectedSchemaNode anyNode, IAObject constant,
+ Map<String, FunctionCallInformation> sourceInformationMap, FunctionCallInformation compareFunctionInfo) {
+ this.type = rename(constant.getType());
+ this.sourceInformationMap = sourceInformationMap;
+
+ sourceInformationMap.put(type.getTypeName(), compareFunctionInfo);
+ return (ARecordType) anyNode.accept(this, anyNode);
+ }
+
+ @Override
+ public IAType visit(RootExpectedSchemaNode node, IExpectedSchemaNode arg) {
+ type = getRecordType(node, type, arg, getTypeName());
+ return type;
+ }
+
+ @Override
+ public IAType visit(ObjectExpectedSchemaNode node, IExpectedSchemaNode arg) {
+ type = getRecordType(node, type, arg, getTypeName());
+ sourceInformationMap.put(type.getTypeName(), createFunctionCallInformation(arg));
+ return node.getParent().accept(this, node);
+ }
+
+ @Override
+ public IAType visit(ArrayExpectedSchemaNode node, IExpectedSchemaNode arg) {
+ type = new AOrderedListType(type, getTypeName());
+ sourceInformationMap.put(type.getTypeName(), createFunctionCallInformation(arg));
+ return node.getParent().accept(this, node);
+ }
+
+ @Override
+ public IAType visit(UnionExpectedSchemaNode node, IExpectedSchemaNode arg) {
+ sourceInformationMap.put(type.getTypeName(), createFunctionCallInformation(arg));
+ return node.getParent().accept(this, arg);
+ }
+
+ @Override
+ public IAType visit(AnyExpectedSchemaNode node, IExpectedSchemaNode arg) {
+ return node.getParent().accept(this, node);
+ }
+
+ private static ARecordType getRecordType(ObjectExpectedSchemaNode objectNode, IAType childType,
+ IExpectedSchemaNode childNode, String typeName) {
+ String key = objectNode.getChildFieldName(childNode);
+ IAType[] fieldTypes = { childType };
+ String[] fieldNames = { key };
+
+ return new ARecordType(typeName, fieldNames, fieldTypes, false);
+ }
+
+ private String getTypeName() {
+ return "FilterPath" + counter++;
+ }
+
+ private FunctionCallInformation createFunctionCallInformation(IExpectedSchemaNode node) {
+ return new FunctionCallInformation(node.getFunctionName(), node.getSourceLocation(),
+ ProjectionFiltrationWarningFactoryProvider.TYPE_MISMATCH_FACTORY);
+ }
+
+ private IAType rename(IAType type) {
+ return new RenamedType(type, getTypeName());
+ }
+
+ private static class RenamedType implements IAType {
+ private static final long serialVersionUID = 992690669300951839L;
+ private final IAType originalType;
+ private final String name;
+
+ RenamedType(IAType originalType, String name) {
+ this.originalType = originalType;
+ this.name = name;
+ }
+
+ @Override
+ public IAType getType() {
+ return originalType.getType();
+ }
+
+ @Override
+ public boolean deepEqual(IAObject obj) {
+ return originalType.deepEqual(obj);
+ }
+
+ @Override
+ public int hash() {
+ return originalType.hash();
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return originalType.getTypeTag();
+ }
+
+ @Override
+ public String getDisplayName() {
+ return originalType.getDisplayName();
+ }
+
+ @Override
+ public String getTypeName() {
+ return name;
+ }
+
+ @Override
+ public <R, T> R accept(IATypeVisitor<R, T> visitor, T arg) {
+ return visitor.visitFlat(this, arg);
+ }
+
+ @Override
+ public ObjectNode toJSON() {
+ return originalType.toJSON();
+ }
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java
index 02307c0..69812a3 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java
@@ -22,6 +22,8 @@
import java.util.Map;
import java.util.Set;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.SourceLocation;
public class ObjectExpectedSchemaNode extends AbstractComplexExpectedSchemaNode {
@@ -37,9 +39,8 @@
return children.entrySet();
}
- public IExpectedSchemaNode addChild(String fieldName, IExpectedSchemaNode child) {
+ public void addChild(String fieldName, IExpectedSchemaNode child) {
children.put(fieldName, child);
- return child;
}
@Override
@@ -54,9 +55,22 @@
@Override
public void replaceChild(IExpectedSchemaNode oldNode, IExpectedSchemaNode newNode) {
+ String fieldName = getChildFieldName(oldNode);
+ children.replace(fieldName, newNode);
+ }
+
+ protected IAType getType(IAType childType, IExpectedSchemaNode childNode, String typeName) {
+ String key = getChildFieldName(childNode);
+ IAType[] fieldTypes = { childType };
+ String[] fieldNames = { key };
+
+ return new ARecordType("typeName", fieldNames, fieldTypes, false);
+ }
+
+ protected String getChildFieldName(IExpectedSchemaNode requestedChild) {
String key = null;
for (Map.Entry<String, IExpectedSchemaNode> child : children.entrySet()) {
- if (child.getValue() == oldNode) {
+ if (child.getValue() == requestedChild) {
key = child.getKey();
break;
}
@@ -64,8 +78,8 @@
if (key == null) {
//this should not happen
- throw new IllegalStateException("Node " + oldNode.getType() + " is not a child");
+ throw new IllegalStateException("Node " + requestedChild.getType() + " is not a child");
}
- children.replace(key, newNode);
+ return key;
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 4ac44b4..55165ed 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -80,6 +80,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -570,6 +571,11 @@
}
@Override
+ public ILogicalOperator visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return visitSingleInputOperator(op);
+ }
+
+ @Override
public ILogicalOperator visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return visitSingleInputOperator(op);
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index d4b7853..cd476f9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -63,6 +63,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -297,6 +298,11 @@
}
@Override
+ public ILogicalOperator visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return visitSingleInputOperator(op);
+ }
+
+ @Override
public ILogicalOperator visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return visitSingleInputOperator(op);
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
index a0bb4b6..8bb7502 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
@@ -48,6 +48,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -167,6 +168,11 @@
}
@Override
+ public Boolean visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return visitInputs(op);
+ }
+
+ @Override
public Boolean visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return visitInputs(op);
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index 8a91059..b47d45d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -51,7 +51,6 @@
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.api.result.ResultSetId;
-import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -137,6 +136,7 @@
private Profile profile;
private ProfileType profileType;
private long totalWarningsCount;
+ private long compileTime;
public long getCount() {
return count;
@@ -187,10 +187,18 @@
public void setProfileType(ProfileType profileType) {
this.profileType = profileType;
}
+
+ public void setCompileTime(long compileTime) {
+ this.compileTime = compileTime;
+ }
+
+ public long getCompileTime() {
+ return compileTime;
+ }
}
class Profile implements Serializable {
- private static final long serialVersionUID = 4813321148252768375L;
+ private static final long serialVersionUID = 4813321148252768376L;
private transient ObjectNode profile;
@@ -200,16 +208,15 @@
private void writeObject(ObjectOutputStream out) throws IOException {
ObjectMapper om = new ObjectMapper();
- out.writeUTF(om.writeValueAsString(profile));
+ byte[] bytes = om.writeValueAsBytes(profile);
+ out.writeInt(bytes.length);
+ out.write(bytes);
}
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
ObjectMapper om = new ObjectMapper();
- JsonNode inNode = om.readTree(in.readUTF());
- if (!inNode.isObject()) {
- throw new IOException("Deserialization error");
- }
- profile = (ObjectNode) inNode;
+ int length = in.readInt();
+ profile = (ObjectNode) om.readTree(in.readNBytes(length));
}
public ObjectNode getProfile() {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
index ddabaa0..0290acf 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -40,6 +40,7 @@
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Expression.Kind;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.base.Literal;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
@@ -1040,6 +1041,13 @@
: super.visit(qe, tupSource);
}
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(IVisitorExtension ve, Mutable<ILogicalOperator> arg)
+ throws CompilationException {
+ // Language extensions should create a child of this class.
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, "Extension dispatch not implemented!");
+ }
+
// At this point "$x in list_expr" is a quantified expression:
// "some $y in list_expr satisfies $x = $y"
// Look for such quantified expression with a constant list_expr ([e1, e2, ... eN])
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index daf0efc..9680667 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-app</artifactId>
<licenses>
@@ -389,7 +389,7 @@
<profile>
<id>asterix-gerrit-asterix-app</id>
<properties>
- <test.excludes>**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/AqlExecutionTest.java,**/*Compression*Test.java,**/*Ssl*Test.java</test.excludes>
+ <test.excludes>**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/AqlExecutionTest.java,**/*Compression*Test.java,**/*Ssl*Test.java,**/Podman*.java</test.excludes>
<itest.excludes>**/*.java</itest.excludes>
</properties>
<build>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index be21673..41be44b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -63,9 +63,7 @@
import org.apache.asterix.lang.common.base.IRewriterFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
-import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
-import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.om.base.IAObject;
@@ -170,10 +168,9 @@
}
}
- public Pair<IReturningStatement, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
- List<ViewDecl> declaredViews, MetadataProvider metadataProvider, IReturningStatement q,
- SessionOutput output, boolean allowNonStoredUdfCalls, boolean inlineUdfsAndViews,
- Collection<VarIdentifier> externalVars, IWarningCollector warningCollector) throws CompilationException {
+ public Pair<IReturningStatement, Integer> reWriteQuery(LangRewritingContext langRewritingContext,
+ IReturningStatement q, SessionOutput output, boolean allowNonStoredUdfCalls, boolean inlineUdfsAndViews,
+ Collection<VarIdentifier> externalVars) throws CompilationException {
if (q == null) {
return null;
}
@@ -182,9 +179,7 @@
generateExpressionTree(q);
}
IQueryRewriter rw = rewriterFactory.createQueryRewriter();
- LangRewritingContext rwCtx = new LangRewritingContext(metadataProvider, declaredFunctions, declaredViews,
- warningCollector, q.getVarCounter());
- rw.rewrite(rwCtx, q, allowNonStoredUdfCalls, inlineUdfsAndViews, externalVars);
+ rw.rewrite(langRewritingContext, q, allowNonStoredUdfCalls, inlineUdfsAndViews, externalVars);
return new Pair<>(q, q.getVarCounter());
}
@@ -299,15 +294,7 @@
boolean printSignature = isQuery && requestParameters != null && requestParameters.isPrintSignature();
- if (isExplainOnly) {
- printPlanAsResult(metadataProvider, output, printer, printSignature);
- if (!conf.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN)) {
- executionPlans.setOptimizedLogicalPlan(null);
- }
- return null;
- }
-
- if (printSignature) {
+ if (printSignature && !isExplainOnly) { //explainOnly adds the signature later
printer.addResultPrinter(SignaturePrinter.newInstance(executionPlans));
}
@@ -338,7 +325,8 @@
if (conf.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN) || isExplainOnly) {
if (isQuery || isLoad) {
- generateOptimizedLogicalPlan(plan, output.config().getPlanFormat(), cboMode);
+ generateOptimizedLogicalPlan(plan, spec.getLogical2PhysicalMap(), output.config().getPlanFormat(),
+ cboMode);
}
}
@@ -541,6 +529,12 @@
.setLogicalPlan(getPrettyPrintVisitor(format).printPlan(plan, printOptimizerEstimates).toString());
}
+ private void generateOptimizedLogicalPlan(ILogicalPlan plan, Map<Object, String> log2phys,
+ SessionConfig.PlanFormat format, boolean printOptimizerEstimates) throws AlgebricksException {
+ executionPlans.setOptimizedLogicalPlan(
+ getPrettyPrintVisitor(format).printPlan(plan, log2phys, printOptimizerEstimates).toString());
+ }
+
private void generateOptimizedLogicalPlan(ILogicalPlan plan, SessionConfig.PlanFormat format,
boolean printOptimizerEstimates) throws AlgebricksException {
executionPlans.setOptimizedLogicalPlan(
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 1ec7405..d2d7a9b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -204,6 +204,7 @@
stats.setJobProfile(responseStats.getJobProfile());
stats.setProcessedObjects(responseStats.getProcessedObjects());
stats.updateTotalWarningsCount(responseStats.getTotalWarningsCount());
+ stats.setCompileTime(responseStats.getCompileTime());
}
private static void updatePropertiesFromCC(IStatementExecutor.StatementProperties statementProperties,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index 3d0f7fc..47685d8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -81,7 +81,7 @@
protected final IReceptionist receptionist;
protected Path workingDir;
- protected String sysAuthHeader;
+ private String sysAuthHeader;
private ILibraryManager libraryManager;
private int timeout;
@@ -250,7 +250,7 @@
}
URI downloadURI = createDownloadURI(libraryTempFile);
doCreate(dvAndName.getFirst(), dvAndName.getSecond(), language,
- ExternalLibraryUtils.digestToHexString(digest), downloadURI, true, sysAuthHeader,
+ ExternalLibraryUtils.digestToHexString(digest), downloadURI, true, getSysAuthHeader(),
requestReference, request);
} else if (op == LibraryOperation.DELETE) {
//DELETE semantics imply ifExists
@@ -281,6 +281,10 @@
}
}
+ protected String getSysAuthHeader() {
+ return sysAuthHeader;
+ }
+
private void writeException(Exception e, IServletResponse response) {
response.setStatus(toHttpErrorStatus(e));
PrintWriter responseWriter = response.writer();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
index bfebfd6..d9df051 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
@@ -102,7 +102,7 @@
printer.printResults();
ResponseMetrics metrics = ResponseMetrics.of(System.nanoTime() - elapsedStart,
metadata.getJobDuration(), stats.getCount(), stats.getSize(), metadata.getProcessedObjects(), 0,
- metadata.getTotalWarningsCount());
+ metadata.getTotalWarningsCount(), stats.getCompileTime());
printer.addFooterPrinter(new MetricsPrinter(metrics, HttpUtil.getPreferredCharset(request)));
if (metadata.getJobProfile() != null) {
printer.addFooterPrinter(new ProfilePrinter(metadata.getJobProfile()));
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 9f83aa8..e14c185 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -357,9 +357,9 @@
// in case of ASYNC delivery, the status is printed by query translator
responsePrinter.addFooterPrinter(new StatusPrinter(executionState.getResultStatus()));
}
- final ResponseMetrics metrics =
- ResponseMetrics.of(System.nanoTime() - elapsedStart, executionState.duration(), stats.getCount(),
- stats.getSize(), stats.getProcessedObjects(), errorCount, stats.getTotalWarningsCount());
+ final ResponseMetrics metrics = ResponseMetrics.of(System.nanoTime() - elapsedStart, executionState.duration(),
+ stats.getCount(), stats.getSize(), stats.getProcessedObjects(), errorCount,
+ stats.getTotalWarningsCount(), stats.getCompileTime());
responsePrinter.addFooterPrinter(new MetricsPrinter(metrics, resultCharset));
if (isPrintingProfile(stats)) {
responsePrinter.addFooterPrinter(new ProfilePrinter(stats.getJobProfile()));
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
index cf2b891..ee018f9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
@@ -37,7 +37,7 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
@@ -50,6 +50,7 @@
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
public class QueryIndexDatasource extends FunctionDataSource {
@@ -73,9 +74,7 @@
ARecordType type = (ARecordType) iType;
IAType[] fieldTypes = type.getFieldTypes();
schemaTypes = new IAType[fieldTypes.length];
- for (int i = 0; i < schemaTypes.length; i++) {
- schemaTypes[i] = fieldTypes[i];
- }
+ System.arraycopy(fieldTypes, 0, schemaTypes, 0, schemaTypes.length);
}
@Override
@@ -103,9 +102,11 @@
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException {
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException {
return metadataProvider.buildBtreeRuntime(jobSpec, opSchema, typeEnv, context, true, false, null, ds, indexName,
- null, null, true, true, false, null, null, null, tupleFilterFactory, outputLimit, false, false);
+ null, null, true, true, false, null, null, null, tupleFilterFactory, outputLimit, false, false,
+ DefaultTupleProjectorFactory.INSTANCE);
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
index 4850bf6..82c70a3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
@@ -25,6 +25,7 @@
import java.util.Objects;
import java.util.Optional;
+import org.apache.asterix.column.ColumnManagerFactory;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
import org.apache.asterix.common.context.DatasetInfoProvider;
@@ -103,6 +104,7 @@
import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.storage.am.common.data.PointablePrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.freepage.AppendOnlyLinkedMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.dataflow.LSMColumnBTreeLocalResource;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeLocalResource;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyLocalResource;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResource;
@@ -162,6 +164,7 @@
registeredClasses.put("ExternalBTreeLocalResource", ExternalBTreeLocalResource.class);
registeredClasses.put("ExternalBTreeWithBuddyLocalResource", ExternalBTreeWithBuddyLocalResource.class);
registeredClasses.put("ExternalRTreeLocalResource", ExternalRTreeLocalResource.class);
+ registeredClasses.put("LSMColumnBTreeLocalResource", LSMColumnBTreeLocalResource.class);
// ILSMMergePolicyFactory
registeredClasses.put("NoMergePolicyFactory", NoMergePolicyFactory.class);
@@ -305,6 +308,9 @@
//External Libraries
registeredClasses.put("LibraryDescriptor", LibraryDescriptor.class);
+
+ //IColumnManagerFactory
+ registeredClasses.put("ColumnManagerFactory", ColumnManagerFactory.class);
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
index 25e51bb..5d74c13 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
@@ -26,6 +26,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -59,6 +60,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -197,6 +199,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
visit(op);
return null;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
index 024a13e..af383c3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
@@ -24,6 +24,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -57,6 +58,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -193,6 +195,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
visitInternal(op, true);
return null;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
index 201a470..66d81d2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
@@ -82,7 +82,7 @@
for (JobletProfile jp : jobletProfiles) {
final Collection<TaskProfile> jobletTasksProfile = jp.getTaskProfiles().values();
for (TaskProfile tp : jobletTasksProfile) {
- processedObjects += tp.getStatsCollector().getAggregatedStats().getTupleCounter().get();
+ processedObjects += tp.getStatsCollector().getAggregatedStats().getInputTupleCounter().get();
aggregateTotalWarningsCount += tp.getTotalWarningsCount();
Set<Warning> taskWarnings = tp.getWarnings();
if (AggregateWarnings.size() < maxWarnings && !taskWarnings.isEmpty()) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
index 8dfe923..d55bb07 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
@@ -28,12 +28,13 @@
private long errorCount;
private long warnCount;
private long diskIoCount;
+ private long compileTime;
private ResponseMetrics() {
}
public static ResponseMetrics of(long elapsedTime, long executionTime, long resultCount, long resultSize,
- long processedObjects, long errorCount, long warnCount) {
+ long processedObjects, long errorCount, long warnCount, long compileTime) {
ResponseMetrics metrics = new ResponseMetrics();
metrics.elapsedTime = elapsedTime;
metrics.executionTime = executionTime;
@@ -42,6 +43,7 @@
metrics.processedObjects = processedObjects;
metrics.errorCount = errorCount;
metrics.warnCount = warnCount;
+ metrics.compileTime = compileTime;
return metrics;
}
@@ -72,4 +74,8 @@
public long getWarnCount() {
return warnCount;
}
+
+ public long getCompileTime() {
+ return compileTime;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
index 5549683..6b1d670 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
@@ -32,6 +32,7 @@
public enum Metrics {
ELAPSED_TIME("elapsedTime"),
EXECUTION_TIME("executionTime"),
+ COMPILE_TIME("compileTime"),
RESULT_COUNT("resultCount"),
RESULT_SIZE("resultSize"),
ERROR_COUNT("errorCount"),
@@ -71,6 +72,8 @@
ResultUtil.printField(pw, Metrics.EXECUTION_TIME.str(),
Duration.formatNanos(metrics.getExecutionTime(), useAscii));
pw.print("\n\t");
+ ResultUtil.printField(pw, Metrics.COMPILE_TIME.str(), Duration.formatNanos(metrics.getCompileTime(), useAscii));
+ pw.print("\n\t");
ResultUtil.printField(pw, Metrics.RESULT_COUNT.str(), metrics.getResultCount(), true);
pw.print("\n\t");
ResultUtil.printField(pw, Metrics.RESULT_SIZE.str(), metrics.getResultSize(), true);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index b017648..775672c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -23,7 +23,6 @@
import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import static org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS;
-import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.rmi.RemoteException;
@@ -76,6 +75,7 @@
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.DatasetConfig.TransactionState;
import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -110,6 +110,7 @@
import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.statement.AdapterDropStatement;
import org.apache.asterix.lang.common.statement.AnalyzeDropStatement;
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
@@ -156,7 +157,6 @@
import org.apache.asterix.lang.common.statement.TypeDropStatement;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.statement.ViewDropStatement;
-import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.common.util.FunctionUtil;
@@ -166,6 +166,7 @@
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.dataset.hints.DatasetHints;
import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetNodegroupCardinalityHint;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -243,7 +244,6 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import org.apache.hyracks.algebricks.data.IAWriterFactory;
import org.apache.hyracks.api.client.IClusterInfoCollector;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -251,7 +251,6 @@
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.api.io.FileSplit;
-import org.apache.hyracks.api.io.UnmanagedFileSplit;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
@@ -440,6 +439,9 @@
handleViewDropStatement(metadataProvider, stmt);
break;
case LOAD:
+ if (stats.getProfileType() == Stats.ProfileType.FULL) {
+ this.jobFlags.add(JobFlag.PROFILE_RUNTIME);
+ }
handleLoadStatement(metadataProvider, stmt, hcc);
break;
case INSERT:
@@ -450,6 +452,9 @@
|| resultDelivery == ResultDelivery.DEFERRED);
metadataProvider.setMaxResultReads(maxResultReads);
}
+ if (stats.getProfileType() == Stats.ProfileType.FULL) {
+ this.jobFlags.add(JobFlag.PROFILE_RUNTIME);
+ }
handleInsertUpsertStatement(metadataProvider, stmt, hcc, resultSet, resultDelivery, outMetadata,
stats, requestParameters, stmtParams, stmtRewriter);
break;
@@ -503,9 +508,6 @@
case EXTERNAL_DATASET_REFRESH:
handleExternalDatasetRefreshStatement(metadataProvider, stmt, hcc);
break;
- case WRITE:
- //Deprecated.
- break;
case FUNCTION_DECL:
handleDeclareFunctionStatement(metadataProvider, stmt);
break;
@@ -561,16 +563,11 @@
config.put(pname, pvalue);
}
- protected Pair<IAWriterFactory, FileSplit> handleWriteStatement(Statement stmt)
- throws InstantiationException, IllegalAccessException, ClassNotFoundException {
- WriteStatement ws = (WriteStatement) stmt;
- File f = new File(ws.getFileName());
- FileSplit outputFile = new UnmanagedFileSplit(ws.getNcName().getValue(), f.getPath());
- IAWriterFactory writerFactory = null;
- if (ws.getWriterClassName() != null) {
- writerFactory = (IAWriterFactory) Class.forName(ws.getWriterClassName()).newInstance();
- }
- return new Pair<>(writerFactory, outputFile);
+ protected LangRewritingContext createLangRewritingContext(MetadataProvider metadataProvider,
+ List<FunctionDecl> declaredFunctions, List<ViewDecl> declaredViews, IWarningCollector warningCollector,
+ int varCounter) {
+ return new LangRewritingContext(metadataProvider, declaredFunctions, declaredViews, warningCollector,
+ varCounter);
}
protected Dataverse handleUseDataverseStatement(MetadataProvider metadataProvider, Statement stmt)
@@ -659,7 +656,7 @@
protected static void validateCompactionPolicy(String compactionPolicy,
Map<String, String> compactionPolicyProperties, MetadataTransactionContext mdTxnCtx,
- boolean isExternalDataset, SourceLocation sourceLoc) throws CompilationException, Exception {
+ boolean isExternalDataset, SourceLocation sourceLoc) throws Exception {
CompactionPolicy compactionPolicyEntity = MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx,
MetadataConstants.METADATA_DATAVERSE_NAME, compactionPolicy);
if (compactionPolicyEntity == null) {
@@ -760,6 +757,10 @@
Dataset dataset = null;
Datatype itemTypeEntity = null, metaItemTypeEntity = null;
boolean itemTypeAdded = false, metaItemTypeAdded = false;
+
+ StorageProperties storageProperties = metadataProvider.getStorageProperties();
+ DatasetFormatInfo datasetFormatInfo = dd.getDatasetFormatInfo(storageProperties.getColumnMaxTupleCount(),
+ storageProperties.getColumnFreeSpaceTolerance());
try {
// Check if the dataverse exists
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
@@ -866,7 +867,7 @@
dataset = new Dataset(dataverseName, datasetName, itemTypeDataverseName, itemTypeName,
metaItemTypeDataverseName, metaItemTypeName, ngName, compactionPolicy, compactionPolicyProperties,
datasetDetails, dd.getHints(), dsType, DatasetIdFactory.generateDatasetId(),
- MetadataUtil.PENDING_ADD_OP, compressionScheme);
+ MetadataUtil.PENDING_ADD_OP, compressionScheme, datasetFormatInfo);
MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset);
if (itemTypeIsInline) {
@@ -1448,15 +1449,12 @@
"full-text filter type is null");
}
- switch (filterType) {
- case FIELD_TYPE_STOPWORDS: {
- filterDescriptor = new StopwordsFullTextFilterDescriptor(dataverseName,
- stmtCreateFilter.getFilterName(), stmtCreateFilter.getStopwordsList());
- break;
- }
- default:
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, stmtCreateFilter.getSourceLocation(),
- "Unexpected full-text filter type: " + filterType);
+ if (FIELD_TYPE_STOPWORDS.equals(filterType)) {
+ filterDescriptor = new StopwordsFullTextFilterDescriptor(dataverseName, stmtCreateFilter.getFilterName(),
+ stmtCreateFilter.getStopwordsList());
+ } else {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, stmtCreateFilter.getSourceLocation(),
+ "Unexpected full-text filter type: " + filterType);
}
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
@@ -1530,8 +1528,7 @@
}
}
- ImmutableList.Builder<IFullTextFilterDescriptor> filterDescriptorsBuilder =
- ImmutableList.<IFullTextFilterDescriptor> builder();
+ ImmutableList.Builder<IFullTextFilterDescriptor> filterDescriptorsBuilder = ImmutableList.builder();
for (String filterName : filterNames) {
FullTextFilterMetadataEntity filterMetadataEntity =
MetadataManager.INSTANCE.getFullTextFilter(mdTxnCtx, dataverseName, filterName);
@@ -2754,8 +2751,10 @@
IQueryRewriter queryRewriter = rewriterFactory.createQueryRewriter();
Query wrappedQuery = queryRewriter.createViewAccessorQuery(viewDecl);
metadataProvider.setDefaultDataverse(dv);
- apiFramework.reWriteQuery(declaredFunctions, Collections.singletonList(viewDecl), metadataProvider,
- wrappedQuery, sessionOutput, false, false, Collections.emptyList(), warningCollector);
+ LangRewritingContext langRewritingContext = createLangRewritingContext(metadataProvider, declaredFunctions,
+ Collections.singletonList(viewDecl), warningCollector, wrappedQuery.getVarCounter());
+ apiFramework.reWriteQuery(langRewritingContext, wrappedQuery, sessionOutput, false, false,
+ Collections.emptyList());
List<List<Triple<DataverseName, String, String>>> dependencies =
ViewUtil.getViewDependencies(viewDecl, foreignKeys, queryRewriter);
@@ -3029,8 +3028,10 @@
fdList.addAll(declaredFunctions);
fdList.add(fd);
metadataProvider.setDefaultDataverse(dv);
- apiFramework.reWriteQuery(fdList, null, metadataProvider, wrappedQuery, sessionOutput, false, false,
- Collections.emptyList(), warningCollector);
+ LangRewritingContext langRewritingContext = createLangRewritingContext(metadataProvider, fdList, null,
+ warningCollector, wrappedQuery.getVarCounter());
+ apiFramework.reWriteQuery(langRewritingContext, wrappedQuery, sessionOutput, false, false,
+ Collections.emptyList());
List<List<Triple<DataverseName, String, String>>> dependencies =
FunctionUtil.getFunctionDependencies(fd, queryRewriter);
@@ -3768,8 +3769,10 @@
Map<VarIdentifier, IAObject> externalVars = createExternalVariables(query, stmtParams);
// Query Rewriting (happens under the same ongoing metadata transaction)
- Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(declaredFunctions, null,
- metadataProvider, query, sessionOutput, true, true, externalVars.keySet(), warningCollector);
+ LangRewritingContext langRewritingContext = createLangRewritingContext(metadataProvider, declaredFunctions,
+ null, warningCollector, query.getVarCounter());
+ Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(langRewritingContext, query,
+ sessionOutput, true, true, externalVars.keySet());
// Query Compilation (happens under the same ongoing metadata transaction)
return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, (Query) rewrittenResult.first,
@@ -3777,7 +3780,7 @@
responsePrinter, warningCollector, requestParameters);
}
- private JobSpecification rewriteCompileInsertUpsert(IClusterInfoCollector clusterInfoCollector,
+ protected JobSpecification rewriteCompileInsertUpsert(IClusterInfoCollector clusterInfoCollector,
MetadataProvider metadataProvider, InsertStatement insertUpsert, Map<String, IAObject> stmtParams)
throws AlgebricksException, ACIDException {
SourceLocation sourceLoc = insertUpsert.getSourceLocation();
@@ -3785,8 +3788,10 @@
Map<VarIdentifier, IAObject> externalVars = createExternalVariables(insertUpsert, stmtParams);
// Insert/upsert statement rewriting (happens under the same ongoing metadata transaction)
- Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(declaredFunctions, null,
- metadataProvider, insertUpsert, sessionOutput, true, true, externalVars.keySet(), warningCollector);
+ LangRewritingContext langRewritingContext = createLangRewritingContext(metadataProvider, declaredFunctions,
+ null, warningCollector, insertUpsert.getVarCounter());
+ Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(langRewritingContext,
+ insertUpsert, sessionOutput, true, true, externalVars.keySet());
InsertStatement rewrittenInsertUpsert = (InsertStatement) rewrittenResult.first;
DataverseName dataverseName = getActiveDataverseName(rewrittenInsertUpsert.getDataverseName());
@@ -3963,9 +3968,8 @@
ActiveEntityEventsListener listener =
(ActiveEntityEventsListener) activeNotificationHandler.getListener(feedId);
if (listener != null && listener.getState() != ActivityState.STOPPED) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Feed " + feedId + " is currently active and connected to the following " + dataset(PLURAL) + "\n"
- + listener.toString());
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Feed " + feedId
+ + " is currently active and connected to the following " + dataset(PLURAL) + "\n" + listener);
} else if (listener != null) {
listener.unregister();
}
@@ -4672,6 +4676,7 @@
}
};
final IStatementCompiler compiler = () -> {
+ long compileStart = System.nanoTime();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
@@ -4682,6 +4687,7 @@
stats.updateTotalWarningsCount(warningCollector.getTotalWarningsCount());
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ stats.setCompileTime(System.nanoTime() - compileStart);
bActiveTxn = false;
return query.isExplain() || isCompileOnly() ? null : jobSpec;
} catch (Exception e) {
@@ -5228,7 +5234,6 @@
case DATAVERSE_DECL:
case FUNCTION_DECL:
case SET:
- case WRITE:
return false;
default:
return true;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
index 156b78a..48cf511 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
@@ -20,7 +20,7 @@
import static java.util.regex.Pattern.CASE_INSENSITIVE;
import static java.util.regex.Pattern.DOTALL;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
import java.util.regex.Pattern;
diff --git a/asterixdb/asterix-app/src/main/resources/entrypoint.py b/asterixdb/asterix-app/src/main/resources/entrypoint.py
index 7bad7ef..918596c 100755
--- a/asterixdb/asterix-app/src/main/resources/entrypoint.py
+++ b/asterixdb/asterix-app/src/main/resources/entrypoint.py
@@ -168,6 +168,7 @@
def quit(self):
self.alive = False
+ self.disconnect_sock()
return True
def handle_call(self):
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 86d8ed4..e8c2c1d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -43,6 +43,7 @@
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.hyracks.bootstrap.CCApplication;
import org.apache.asterix.hyracks.bootstrap.NCApplication;
+import org.apache.asterix.lang.common.util.ExpressionUtils;
import org.apache.asterix.test.dataflow.TestLsmIoOpCallbackFactory;
import org.apache.asterix.test.dataflow.TestPrimaryIndexOperationTrackerFactory;
import org.apache.commons.io.FileUtils;
@@ -132,13 +133,13 @@
cc = new ClusterControllerService(ccConfig, ccApplication);
nodeNames = ccConfig.getConfigManager().getNodeNames();
- if (deleteOldInstanceData) {
+ if (deleteOldInstanceData && nodeNames != null) {
deleteTransactionLogs();
removeTestStorageFiles();
deleteCCFiles();
}
final List<NodeControllerService> nodeControllers = new ArrayList<>();
- for (String nodeId : nodeNames) {
+ for (String nodeId : ExpressionUtils.emptyIfNull(nodeNames)) {
// mark this NC as virtual, so that the CC doesn't try to start via NCService...
configManager.set(nodeId, NCConfig.Option.NCSERVICE_PORT, NCConfig.NCSERVICE_PORT_DISABLED);
final INCApplication ncApplication = createNCApplication();
@@ -303,7 +304,7 @@
stopCC(false);
- if (deleteOldInstanceData) {
+ if (deleteOldInstanceData && nodeNames != null) {
deleteTransactionLogs();
removeTestStorageFiles();
deleteCCFiles();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ProfilingTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ProfilingTestExecutor.java
new file mode 100644
index 0000000..44502da
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ProfilingTestExecutor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.common;
+
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.function.Predicate;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.asterix.testframework.xml.ParameterTypeEnum;
+import org.apache.asterix.testframework.xml.TestCase;
+
+public class ProfilingTestExecutor extends TestExecutor {
+
+ private final TestCase.CompilationUnit.Parameter profile = new TestCase.CompilationUnit.Parameter();
+
+ public InputStream executeQueryService(String str, TestCaseContext.OutputFormat fmt, URI uri,
+ List<TestCase.CompilationUnit.Parameter> params, boolean jsonEncoded, Charset responseCharset,
+ Predicate<Integer> responseCodeValidator, boolean cancellable) throws Exception {
+ profile.setName("profile");
+ profile.setValue("timings");
+ profile.setType(ParameterTypeEnum.STRING);
+ params.add(profile);
+ return super.executeQueryService(str, fmt, uri, constructQueryParameters(str, fmt, params), jsonEncoded,
+ responseCharset, responseCodeValidator, cancellable);
+
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/lazy/LazyVisitablePointableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/lazy/LazyVisitablePointableTest.java
new file mode 100644
index 0000000..d981be1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/lazy/LazyVisitablePointableTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.om.lazy;
+
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.asterix.external.parser.JSONDataParser;
+import org.apache.asterix.om.lazy.AbstractLazyVisitablePointable;
+import org.apache.asterix.om.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.lazy.TypedRecordLazyVisitablePointable;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.pointables.cast.ACastVisitor;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.comparisons.DeepEqualAssessor;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.fasterxml.jackson.core.JsonFactory;
+
+/**
+ * Test the operations of {@link AbstractLazyVisitablePointable}
+ */
+public class LazyVisitablePointableTest {
+ private static final String BASE_DIR;
+ private static final String[] FILE_PATHS;
+ private final JSONDataParser parser;
+ private final ACastVisitor castVisitor;
+ private final RecordTypeInference schemaInference;
+ private final DeepEqualAssessor deepEqualAssessor;
+ private final RecordLazyVisitablePointable openLazyPointable;
+ private final ARecordVisitablePointable openPointable;
+ private final ArrayBackedValueStorage recordStorage;
+ private final Triple<IVisitablePointable, IAType, Boolean> arg;
+
+ static {
+ BASE_DIR = "data";
+ FILE_PATHS = new String[] { joinPath(BASE_DIR, "hdfs", "parquet", "dummy_tweet.json"),
+ joinPath(BASE_DIR, "nested01", "person2.adm"), joinPath(BASE_DIR, "yelp-checkin", "use-case-1.json"),
+ joinPath(BASE_DIR, "yelp-checkin", "use-case-2.json"),
+ joinPath(BASE_DIR, "yelp-checkin", "use-case-3.json"),
+ joinPath(BASE_DIR, "yelp-checkin", "use-case-4.json") };
+ }
+
+ public LazyVisitablePointableTest() {
+ parser = new JSONDataParser(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, new JsonFactory());
+ castVisitor = new ACastVisitor();
+ schemaInference = new RecordTypeInference();
+ deepEqualAssessor = new DeepEqualAssessor();
+ openLazyPointable = new RecordLazyVisitablePointable(true);
+ openPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+ recordStorage = new ArrayBackedValueStorage();
+ arg = new Triple<>(null, null, null);
+ arg.third = Boolean.FALSE;
+ }
+
+ private void prepareParser(String path) throws IOException {
+ FileInputStream inputStream = new FileInputStream(path);
+ parser.setInputStream(inputStream);
+ }
+
+ private void inferCastAndCompare() throws HyracksDataException {
+ recordStorage.reset();
+ while (parser.parse(recordStorage.getDataOutput())) {
+ openLazyPointable.set(recordStorage);
+
+ //Infer the schema
+ ARecordType inferredFromOpen = (ARecordType) openLazyPointable.accept(schemaInference, "fromOpen");
+ ARecordVisitablePointable closedPointable = new ARecordVisitablePointable(inferredFromOpen);
+ arg.first = closedPointable;
+ arg.second = inferredFromOpen;
+
+ //Cast to closed using the inferred type
+ openPointable.set(recordStorage);
+ openPointable.accept(castVisitor, arg);
+ //Ensure both closed and open records are the same
+ Assert.assertTrue(deepEqualAssessor.isEqual(openPointable, closedPointable));
+
+ //Ensure lazy pointable can handle closed types
+ TypedRecordLazyVisitablePointable closedLazyPointable =
+ new TypedRecordLazyVisitablePointable(inferredFromOpen);
+ closedLazyPointable.set(closedPointable);
+ //Infer the type (again) but from a closed type
+ ARecordType inferredFromClosed = (ARecordType) closedLazyPointable.accept(schemaInference, "fromClosed");
+ //Ensure both inferred types are the same
+ Assert.assertTrue(inferredFromOpen.deepEqual(inferredFromClosed));
+ recordStorage.reset();
+ }
+ }
+
+ @Test
+ public void runTest() throws IOException {
+ for (String path : FILE_PATHS) {
+ prepareParser(path);
+ inferCastAndCompare();
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/lazy/RecordTypeInference.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/lazy/RecordTypeInference.java
new file mode 100644
index 0000000..4a1a4a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/lazy/RecordTypeInference.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.om.lazy;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.asterix.om.lazy.AbstractListLazyVisitablePointable;
+import org.apache.asterix.om.lazy.FlatLazyVisitablePointable;
+import org.apache.asterix.om.lazy.ILazyVisitablePointableVisitor;
+import org.apache.asterix.om.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+
+/**
+ * Infers the type of single record using lazy visitable pointable
+ */
+public class RecordTypeInference implements ILazyVisitablePointableVisitor<IAType, String> {
+ private final ByteArrayAccessibleInputStream in;
+ private final DataInputStream dataIn;
+ private final UTF8StringReader utf8Reader;
+
+ public RecordTypeInference() {
+ in = new ByteArrayAccessibleInputStream(new byte[] {}, 0, 0);
+ dataIn = new DataInputStream(in);
+ utf8Reader = new UTF8StringReader();
+ }
+
+ @Override
+ public IAType visit(RecordLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+ String[] fieldNames = new String[pointable.getNumberOfChildren()];
+ IAType[] fieldTypes = new IAType[pointable.getNumberOfChildren()];
+ for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
+ pointable.nextChild();
+ fieldNames[i] = deserializeString(pointable.getFieldName());
+ fieldTypes[i] = pointable.getChildVisitablePointable().accept(this, fieldNames[i]);
+ }
+ // isOpen has to be false here to ensure that every field go to the closed part
+ return new ARecordType(arg, fieldNames, fieldTypes, false);
+ }
+
+ @Override
+ public IAType visit(AbstractListLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+ IAType itemType = BuiltinType.ANY;
+ String itemTypeName = arg + "Item";
+ for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
+ pointable.nextChild();
+ IAType ithItemType = pointable.getChildVisitablePointable().accept(this, itemTypeName);
+ if (itemType.getTypeTag() != ATypeTag.ANY && itemType.getTypeTag() != ithItemType.getTypeTag()) {
+ throw new UnsupportedOperationException("Union types are not supported");
+ }
+ itemType = ithItemType;
+ }
+ return pointable.getTypeTag() == ATypeTag.ARRAY ? new AOrderedListType(itemType, arg)
+ : new AUnorderedListType(itemType, arg);
+ }
+
+ @Override
+ public IAType visit(FlatLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+ return BuiltinType.getBuiltinType(pointable.getTypeTag());
+ }
+
+ private String deserializeString(IValueReference stringValue) throws HyracksDataException {
+ in.setContent(stringValue.getByteArray(), stringValue.getStartOffset(), stringValue.getLength());
+ try {
+ return UTF8StringUtil.readUTF8(dataIn, utf8Reader);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index 408882d..d704d8e 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -33,6 +33,7 @@
import java.util.List;
import org.apache.asterix.app.external.ExternalUDFLibrarian;
+import org.apache.asterix.app.external.IExternalUDFLibrarian;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
@@ -65,11 +66,17 @@
}
public static void setUp(String configFile, TestExecutor executor, boolean startHdfs) throws Exception {
+ setUp(configFile, executor, startHdfs, false, new ExternalUDFLibrarian());
+ }
+
+ public static void setUp(String configFile, TestExecutor executor, boolean startHdfs, boolean disableLangExec,
+ IExternalUDFLibrarian librarian) throws Exception {
testExecutor = executor;
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
- ExecutionTestUtil.setUp(cleanupOnStart, configFile, integrationUtil, startHdfs, null);
- librarian = new ExternalUDFLibrarian();
+ if (!disableLangExec) {
+ ExecutionTestUtil.setUp(cleanupOnStart, configFile, integrationUtil, startHdfs, null);
+ }
testExecutor.setLibrarian(librarian);
if (repeat != 1) {
System.out.println("FYI: each test will be run " + repeat + " times.");
@@ -151,7 +158,9 @@
NodeControllerService[] ncs = integrationUtil.ncs;
// Checks that dataset files are uniformly distributed across each io device.
for (NodeControllerService nc : ncs) {
- checkNcStore(nc);
+ if (nc != null) {
+ checkNcStore(nc);
+ }
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/MetricsExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/MetricsExecutionTest.java
index 93031c5..6eaa09d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/MetricsExecutionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/MetricsExecutionTest.java
@@ -31,7 +31,7 @@
import org.junit.runners.Parameterized.Parameters;
/**
- * Runs the cluster state runtime tests with the storage parallelism.
+ * Runs the cluster runtime tests and checks the query metrics.
*/
@RunWith(Parameterized.class)
public class MetricsExecutionTest {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppBatchPointLookupExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppBatchPointLookupExecutionTest.java
new file mode 100644
index 0000000..2608447
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppBatchPointLookupExecutionTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.runtime;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the SQL++ with Batched point-lookups enabled.
+ */
+@RunWith(Parameterized.class)
+public class SqlppBatchPointLookupExecutionTest {
+ protected static final String TEST_CONFIG_FILE_NAME = "src/test/resources/cc-batch-lookup.conf";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ final TestExecutor testExecutor = new TestExecutor();
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor);
+ setNcEndpoints(testExecutor);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "SqlppBatchPointLookupExecutionTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_batch_lookup.xml", "testsuite_sqlpp_batch_lookup.xml");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public SqlppBatchPointLookupExecutionTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ LangExecutionUtil.test(tcCtx);
+ }
+
+ private static void setNcEndpoints(TestExecutor testExecutor) {
+ final NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs;
+ final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
+ final String ip = InetAddress.getLoopbackAddress().getHostAddress();
+ for (NodeControllerService nc : ncs) {
+ final String nodeId = nc.getId();
+ final INcApplicationContext appCtx = (INcApplicationContext) nc.getApplicationContext();
+ int apiPort = appCtx.getExternalProperties().getNcApiPort();
+ ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip, apiPort));
+ }
+ testExecutor.setNcEndPoints(ncEndPoints);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppProfiledExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppProfiledExecutionTest.java
new file mode 100644
index 0000000..b056d2c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppProfiledExecutionTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.runtime;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.test.common.ProfilingTestExecutor;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the SQL++ runtime tests with full runtime profiling.
+ */
+@RunWith(Parameterized.class)
+public class SqlppProfiledExecutionTest {
+ protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ final TestExecutor testExecutor = new ProfilingTestExecutor();
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor);
+ setNcEndpoints(testExecutor);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "SqlppProfiledExecutionTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_sqlpp.xml", "testsuite_sqlpp_profiled.xml");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public SqlppProfiledExecutionTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ LangExecutionUtil.test(tcCtx);
+ }
+
+ private static void setNcEndpoints(TestExecutor testExecutor) {
+ final NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs;
+ final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
+ final String ip = InetAddress.getLoopbackAddress().getHostAddress();
+ for (NodeControllerService nc : ncs) {
+ final String nodeId = nc.getId();
+ final INcApplicationContext appCtx = (INcApplicationContext) nc.getApplicationContext();
+ int apiPort = appCtx.getExternalProperties().getNcApiPort();
+ ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip, apiPort));
+ }
+ testExecutor.setNcEndPoints(ncEndPoints);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
index 8a87de7..5d10028 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
@@ -279,6 +279,7 @@
invokeMethod(rewriter, "rewriteGroupBys");
invokeMethod(rewriter, "rewriteSetOperations");
invokeMethod(rewriter, "inlineColumnAlias");
+ invokeMethod(rewriter, "rewriteSelectExcludeSugar");
invokeMethod(rewriter, "rewriteWindowExpressions");
invokeMethod(rewriter, "rewriteGroupingSets");
invokeMethod(rewriter, "variableCheckAndRewrite");
diff --git a/asterixdb/asterix-app/src/test/resources/cc-batch-lookup.conf b/asterixdb/asterix-app/src/test/resources/cc-batch-lookup.conf
new file mode 100644
index 0000000..6e10481
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/cc-batch-lookup.conf
@@ -0,0 +1,65 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=target/tmp/asterix_nc1/iodevice1,
+iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+credential.file=src/test/resources/security/passwd
+python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.pagesize=32KB
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=512MB
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+credential.file=src/test/resources/security/passwd
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+compiler.batch.lookup=true
+messaging.frame.size=4096
+messaging.frame.count=512
+
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.1.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.1.sqlpp
index 3c1a24d..76f8416 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.1.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Index on top-level field, single OBJECT_ADD function application.
+ * Index "usersNameIdx" should be used.
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name : string );
+
+FROM Test.Users U
+LET augmentedUser = OBJECT_ADD(U, "favoriteColor", "Green")
+WHERE augmentedUser.name = "Glenn"
+SELECT *;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.10.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.10.sqlpp
new file mode 100644
index 0000000..7e018b9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.10.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Index on top-level field, three OBJECT_ADD function applications.
+ * Primary index should used w/ INLJ.
+ */
+
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+
+FROM Test.Users U1, Test.Users U2
+LET augmentedUser1 = OBJECT_ADD(U1, "favoriteColor", "Green"),
+ augmentedUser2 = OBJECT_ADD(U2, "favoriteFood", "Pizza"),
+ augmentedUser3 = OBJECT_ADD(augmentedUser2, "favoriteColor", "Red"),
+ augmentedUser4 = OBJECT_ADD(augmentedUser3, "favoriteDrink", "Wine")
+WHERE augmentedUser1.name = "John" AND
+ augmentedUser2.name = "Sally" AND
+ TO_BIGINT(U1.bestFriend) /* +indexnl */ = augmentedUser4.user_id
+SELECT *;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.11.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.11.sqlpp
new file mode 100644
index 0000000..782060f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.11.sqlpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Two indexes on nested fields, one OBJECT_ADD function application each.
+ * Index should be used in both cases.
+ */
+
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint, name: { first: string } };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX firstUsersNameIdx ON Users ( name.first );
+CREATE INDEX lastUsersNameIdx ON Users ( name.last: string );
+
+FROM Test.Users U1
+LET augmentedUser1 = OBJECT_ADD(U1, "favoriteColor", "Green")
+WHERE augmentedUser1.name.first = "Glenn"
+SELECT augmentedUser1.*
+
+UNION ALL
+
+FROM Test.Users U2
+LET augmentedUser2 = OBJECT_ADD(U2, "favoriteFood", "Pizza")
+WHERE augmentedUser2.name.last = "John"
+SELECT augmentedUser2.*;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.2.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.2.sqlpp
new file mode 100644
index 0000000..4583e1c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.2.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Index on top-level field, two OBJECT_ADD function applications.
+ * Index "usersNameIdx" should be used.
+ */
+
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name : string );
+
+FROM Test.Users U
+LET augmentedUser1 = OBJECT_ADD(U, "favoriteColor", "Green"),
+ augmentedUser2 = OBJECT_ADD(augmentedUser1, "favoriteCity", "Irvine")
+WHERE augmentedUser2.name = "Glenn"
+SELECT *;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.3.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.3.sqlpp
index 3c1a24d..0d81c0f 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.3.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Index on nested field, one OBJECT_ADD function application.
+ * Index "usersNameIdx" should be used.
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name.first : string );
+
+FROM Test.Users U
+LET augmentedUser = OBJECT_ADD(U, "favoriteColor", "Green")
+WHERE augmentedUser.name.first = "Glenn"
+SELECT *;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.4.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.4.sqlpp
index 3c1a24d..95a60e6 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.4.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Index on top-level field, one OBJECT_REMOVE function application.
+ * Index "usersNameIdx" should be used.
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name : string );
+
+FROM Test.Users U
+LET augmentedUser = OBJECT_REMOVE(U, "favoriteColor")
+WHERE augmentedUser.name = "Glenn"
+SELECT *;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.5.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.5.sqlpp
index 3c1a24d..41af922 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.5.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Index on top-level field, one OBJECT_PUT function application.
+ * Index should NOT used.
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name : string );
+
+FROM Test.Users U
+LET augmentedUser = OBJECT_PUT(U, "name", "John")
+WHERE augmentedUser.name = "Glenn"
+SELECT *;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.6.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.6.sqlpp
new file mode 100644
index 0000000..8b573b5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.6.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Index on top-level field, one OBJECT_ADD function application.
+ * Primary index should used w/ INLJ.
+ */
+
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+
+FROM Test.Users U1, Test.Users U2
+LET augmentedUser1 = OBJECT_ADD(U1, "favoriteColor", "Green"),
+ augmentedUser2 = OBJECT_ADD(U2, "favoriteFood", "Pizza")
+WHERE augmentedUser1.name = "John" AND
+ augmentedUser2.name = "Sally" AND
+ TO_BIGINT(U1.bestFriend) /* +indexnl */ = augmentedUser2.user_id
+SELECT *;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.7.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.7.sqlpp
index 3c1a24d..cb48c1c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.7.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Index on top-level field, OBJECT_ADD followed by OBJECT_REMOVE.
+ * Index should NOT be used.
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name : string );
+
+FROM Test.Users U
+LET augmentedUser = OBJECT_ADD(OBJECT_REMOVE(U, "name"), "name", "Glenn")
+WHERE augmentedUser.firstName = "Glenn"
+SELECT *;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.8.sqlpp
similarity index 60%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.8.sqlpp
index 3c1a24d..8d780bf 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.8.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Index on top-level field, single OBJECT_CONCAT function application with multiple records.
+ * Index "usersNameIdx" should be used.
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name : string );
+
+FROM Test.Users U
+LET augmentedUser = OBJECT_CONCAT({"favoriteColor": "Green"}, U, {"birthdate": "10/09/1996"})
+WHERE augmentedUser.name = "Glenn"
+SELECT *;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.9.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.9.sqlpp
new file mode 100644
index 0000000..ad827e0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/index-through-object/index-through-object.9.sqlpp
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Index on top-level field, three OBJECT_ADD function applications.
+ * Index should be used twice (and not used for E1).
+ */
+
+DROP DATAVERSE Test IF EXISTS;
+CREATE DATAVERSE Test;
+USE Test;
+
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+CREATE INDEX usersNameIdx ON Users ( name : string );
+
+FROM Test.Users U1, Test.Users U2, [{"name": "Glenn"}] E1
+LET augmentedUser1 = OBJECT_ADD(U1, "favoriteColor", "Green"),
+ augmentedUser2 = OBJECT_ADD(U2, "favoriteFood", "Pizza"),
+ augmentedUser3 = OBJECT_ADD(E1, "favoriteColor", "Blue")
+WHERE augmentedUser1.name = "John" AND
+ augmentedUser2.name = "Sally" AND
+ augmentedUser3.name = "Glenn" AND
+ augmentedUser1.bestFriend = augmentedUser2.user_id
+SELECT *;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
index efd14f7..b596e4b 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
@@ -5,12 +5,12 @@
{
-- AGGREGATE |LOCAL|
-- ASSIGN |LOCAL|
- -- MICRO_PRE_CLUSTERED_GROUP_BY[$$215] |LOCAL|
+ -- MICRO_PRE_CLUSTERED_GROUP_BY[$$230] |LOCAL|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- MICRO_STABLE_SORT [$$215(ASC)] |LOCAL|
+ -- MICRO_STABLE_SORT [$$230(ASC)] |LOCAL|
-- ASSIGN |LOCAL|
-- UNNEST |LOCAL|
-- SUBPLAN |LOCAL|
@@ -27,7 +27,7 @@
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- INSERT_DELETE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$199] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$214] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -35,27 +35,27 @@
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$252] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$267] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$252(ASC)] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$252] |PARTITIONED|
+ -- STABLE_SORT [$$267(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$267] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$251][$$222] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$266][$$237] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- UNNEST |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$313] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$322] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
- -- MICRO_PRE_CLUSTERED_GROUP_BY[$$315, $$316] |LOCAL|
+ -- MICRO_PRE_CLUSTERED_GROUP_BY[$$324, $$325] |LOCAL|
{
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
@@ -65,8 +65,8 @@
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$313(ASC), $$315(ASC), $$316(ASC)] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$313] |PARTITIONED|
+ -- STABLE_SORT [$$322(ASC), $$324(ASC), $$325(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$322] |PARTITIONED|
-- UNION_ALL |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -102,7 +102,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (channels.Reports.Reports) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$259(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$274(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -147,7 +147,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (channels.Reports.Reports) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$259(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$274(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -161,12 +161,12 @@
-- DATASOURCE_SCAN (channels.UserLocations) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$222] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$237] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$233, $$235][$$224, $$225] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$233, $$235] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$248, $$250][$$239, $$240] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$248, $$250] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.1.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.1.plan
new file mode 100644
index 0000000..ff509f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.1.plan
@@ -0,0 +1,18 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$36(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.usersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.10.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.10.plan
new file mode 100644
index 0000000..f5ab2c2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.10.plan
@@ -0,0 +1,20 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$80(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$80] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.11.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.11.plan
new file mode 100644
index 0000000..6971ab4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.11.plan
@@ -0,0 +1,39 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- UNION_ALL |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$78(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.firstUsersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$82(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.lastUsersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.2.plan
new file mode 100644
index 0000000..90816ce
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.2.plan
@@ -0,0 +1,18 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$48(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.usersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.3.plan
new file mode 100644
index 0000000..6930560
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.3.plan
@@ -0,0 +1,18 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$38(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.usersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.4.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.4.plan
new file mode 100644
index 0000000..ff509f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.4.plan
@@ -0,0 +1,18 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$36(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.usersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.5.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.5.plan
new file mode 100644
index 0000000..f0f6a03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.5.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.6.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.6.plan
new file mode 100644
index 0000000..b4aee6b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.6.plan
@@ -0,0 +1,20 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$58(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$58] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.7.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.7.plan
new file mode 100644
index 0000000..f0f6a03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.7.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.8.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.8.plan
new file mode 100644
index 0000000..6930560
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.8.plan
@@ -0,0 +1,18 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$38(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.usersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.9.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.9.plan
new file mode 100644
index 0000000..1d2e55b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/index-through-object/index-through-object.9.plan
@@ -0,0 +1,47 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$82][$$83] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$82] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$106(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.usersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$83] |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$110(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (Test.Users.usersNameIdx) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01.plan
index eed20bb..f94167e 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01.plan
@@ -15,14 +15,14 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01_ps.plan
index 61cf933..9c39aef 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_01_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$23(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$24(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
@@ -28,7 +28,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -50,7 +50,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02.plan
index 80fa709..e20acdb 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02.plan
@@ -15,14 +15,14 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$25(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$26(ASC) ] |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02_ps.plan
index 159cc10..f79649b 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive-open_02_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$25(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$25(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$26(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$26(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
@@ -28,7 +28,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -50,7 +50,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01.plan
index eed20bb..f94167e 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01.plan
@@ -15,14 +15,14 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01_ps.plan
index 61cf933..9c39aef 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_01_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$23(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$24(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
@@ -28,7 +28,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -50,7 +50,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02.plan
index 80fa709..e20acdb 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02.plan
@@ -15,14 +15,14 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$25(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$26(ASC) ] |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02_ps.plan
index 159cc10..f79649b 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/orders-index-search-conjunctive_02_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$25(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$25(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$26(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$26(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
@@ -28,7 +28,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -50,7 +50,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (tpch.Orders.Orders) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/q01_pricing_summary_report_nt_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/q01_pricing_summary_report_nt_ps.plan
index 0128265..b5c8603 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/q01_pricing_summary_report_nt_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/q01_pricing_summary_report_nt_ps.plan
@@ -22,13 +22,13 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$206, $$207] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$217, $$218] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$206, $$207] |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$181, $$182] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$217, $$218] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$192, $$193] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
@@ -51,13 +51,13 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$206, $$207] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$217, $$218] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$206, $$207] |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$181, $$182] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$217, $$218] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$192, $$193] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan
index e0059ed..4e9ed38 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan
@@ -20,12 +20,12 @@
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$44][$$46] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$47][$$49] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- DATASOURCE_SCAN (test.s) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$46] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$49] |PARTITIONED|
-- STREAM_PROJECT |UNPARTITIONED|
-- ASSIGN |UNPARTITIONED|
-- UNNEST |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
index 5d16539..3a4e8a7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
@@ -20,8 +20,8 @@
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$45][$$44] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$45] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$48][$$47] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$48] |PARTITIONED|
-- ASSIGN |UNPARTITIONED|
-- UNNEST |UNPARTITIONED|
-- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
index d61947a..d7bad18 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
@@ -8,7 +8,7 @@
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$173] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$180] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- AGGREGATE |LOCAL|
@@ -18,9 +18,9 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$173][$$215] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$180][$$222] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$157] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$164] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- AGGREGATE |LOCAL|
@@ -28,13 +28,13 @@
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$157(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$164(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$157][$$171] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$164][$$178] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$137] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$144] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- AGGREGATE |LOCAL|
@@ -44,7 +44,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$137][$$147] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$144][$$154] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -60,7 +60,7 @@
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$205] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -73,35 +73,35 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$178] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$220, $$221] |PARTITIONED|
+ -- SORT_GROUP_BY[$$227, $$228] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$220, $$221] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$158, $$161] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$227, $$228] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$165, $$168] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$158(ASC), $$161(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$165(ASC), $$168(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$150][$$149] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$157][$$156] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
-- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -110,7 +110,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$206] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- AGGREGATE |LOCAL|
@@ -120,7 +120,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$206][$$207] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -133,7 +133,7 @@
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$205] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -146,7 +146,7 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$156] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -157,30 +157,30 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$215] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$222] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$226, $$227] |PARTITIONED|
+ -- SORT_GROUP_BY[$$233, $$234] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$226, $$227] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$174, $$177] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$233, $$234] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$181, $$184] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$174(ASC), $$177(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$181(ASC), $$184(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$152][$$151] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$152] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$159][$$158] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$159] |PARTITIONED|
-- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -189,7 +189,7 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$178] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$185] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- AGGREGATE |LOCAL|
@@ -197,18 +197,18 @@
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$178(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$185(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$178][$$181] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$185][$$188] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$206] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- AGGREGATE |LOCAL|
@@ -218,7 +218,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$206][$$207] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -231,7 +231,7 @@
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$205] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -244,31 +244,31 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$181] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$188] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$223, $$224] |PARTITIONED|
+ -- SORT_GROUP_BY[$$230, $$231] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$223, $$224] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$193, $$194] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$230, $$231] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$200, $$201] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$193(ASC), $$194(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$200(ASC), $$201(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$196][$$198] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$204][$$205] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
-- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -277,7 +277,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$206] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- AGGREGATE |LOCAL|
@@ -287,7 +287,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$206][$$207] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -300,7 +300,7 @@
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$205] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -315,7 +315,7 @@
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$205] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -328,7 +328,7 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$158] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.1.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.1.plan
new file mode 100644
index 0000000..f0f6a03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.1.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.2.plan
new file mode 100644
index 0000000..f0f6a03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.2.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.3.plan
new file mode 100644
index 0000000..f0f6a03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.3.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.4.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.4.plan
new file mode 100644
index 0000000..f0f6a03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.4.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.8.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.8.plan
new file mode 100644
index 0000000..f0f6a03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.8.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.9.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.9.plan
new file mode 100644
index 0000000..75a50b4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.9.plan
@@ -0,0 +1,41 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$88][$$89] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$88] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$89] |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01.plan
index 2326c55..e4167f0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01.plan
@@ -15,7 +15,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01_ps.plan
index 4bdc66b..8e1370f 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$23(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$24(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02.plan
index eb1d57c..c87d9ce 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02.plan
@@ -15,7 +15,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$25(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$26(ASC) ] |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02_ps.plan
index dce703e..6db1de8 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$25(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$25(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$26(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$26(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01.plan
index 2326c55..e4167f0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01.plan
@@ -15,7 +15,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01_ps.plan
index 4bdc66b..8e1370f 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$23(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$24(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02.plan
index eb1d57c..c87d9ce 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02.plan
@@ -15,7 +15,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- SORT_MERGE_EXCHANGE [$$25(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$26(ASC) ] |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02_ps.plan
index dce703e..6db1de8 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02_ps.plan
@@ -16,8 +16,8 @@
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$25(ASC)] |PARTITIONED|
- -- RANGE_PARTITION_EXCHANGE [$$25(ASC)] |PARTITIONED|
+ -- STABLE_SORT [$$26(ASC)] |PARTITIONED|
+ -- RANGE_PARTITION_EXCHANGE [$$26(ASC)] |PARTITIONED|
-- FORWARD |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/query_issue849.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/query_issue849.plan
index 0f1879f..59b09a9 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/query_issue849.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/query_issue849.plan
@@ -22,8 +22,8 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- BTREE_SEARCH (test.s.s) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$52(ASC)] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$52] |PARTITIONED|
+ -- STABLE_SORT [$$55(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$55] |PARTITIONED|
-- ASSIGN |UNPARTITIONED|
-- UNNEST |UNPARTITIONED|
-- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/only_batch_lookup.xml b/asterixdb/asterix-app/src/test/resources/runtimets/only_batch_lookup.xml
new file mode 100644
index 0000000..334dd52
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/only_batch_lookup.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="failed">
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.1.ddl.sqlpp
new file mode 100644
index 0000000..601eb0c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.1.ddl.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse TinySocial if exists;
+create dataverse TinySocial;
+
+use TinySocial;
+
+create type addressType as closed {state: string, country: string, zip_code: int?};
+create type openType as {id: int};
+create type closedType as closed {id: int, list_f: [addressType]};
+
+create dataset openDs(openType) primary key id;
+create dataset closedDs(closedType) primary key id;
+
+create type TinySocial.TwitterUserType as
+{
+ `screen-name` : string,
+ lang : string,
+ friends_count : bigint,
+ statuses_count : bigint,
+ name : string,
+ followers_count : bigint
+};
+
+create type TinySocial.TweetMessageType as
+ closed {
+ tweetid : string,
+ user : TwitterUserType,
+ `sender-location` : point?,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string
+};
+
+create type t1 AS {
+
+};
+
+create type t2 AS {
+id: int,
+compType: t1
+};
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid hints (`CARDINALITY`=`100`);
+create dataset d1(t2) primary key id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.2.update.sqlpp
new file mode 100644
index 0000000..7e8e5ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.2.update.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use TinySocial;
+
+load dataset TweetMessages using localfs ((`path`=`asterix_nc1://data/tinysocial/twm.adm`),(`format`=`adm`));
+
+insert into d1([
+{"id":1, "compType":{"sth":33}},
+{"id":2, "compType":{"sth":44}, "followers":["John Green", "Emily Jones"]}
+]);
+
+insert into openDs([
+{"id": 1, "list_f": [ [1,2,1], [9999,3] ]},
+{"id": 2, "list_f": [ ["white","blue","magenta"], ["red", "black"] ]},
+{"id": 3, "list_f": [ 1 , 2 ]},
+{"id": 4, "list_f": [ {"state": "OH", "country": "US"} , {"state": "CA", "country": "US", "zip_code": 92863} ]},
+{"id": 5, "list_f": [ {"state": "OR", "country": "US", "zip_code": null} , {"state": "IL", "country": "US", "zip_code": 92863} ]},
+{"id": 6, "list_f": null},
+{"id": 7}
+]);
+
+insert into closedDs([
+{"id": 1, "list_f": [ {"state": "OH", "country": "US"} , {"state": "CA", "country": "US", "zip_code": 92863} ]},
+{"id": 2, "list_f": [ {"state": "OR", "country": "US", "zip_code": null} , {"state": "IL", "country": "US", "zip_code": 92863} ]}
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.3.query.sqlpp
new file mode 100644
index 0000000..eebfec8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.3.query.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use TinySocial;
+
+FROM [
+ {"id":1, "t1": (select array_binary_search(null, 1))},
+ {"id":2, "t2": (select array_binary_search(missing, 1))},
+ {"id":3, "t3": (select array_binary_search([], 1))},
+ {"id":4, "t4": (select array_binary_search([1,2,3,4,5,6,7], 4))},
+ {"id":5, "t5": (select array_binary_search([1,2,3,4,5,6,7], 1))},
+ {"id":6, "t6": (select array_binary_search([1,2,3,4,5,6,7], 7))},
+ {"id":7, "t7": (select array_binary_search(["a", "b", "c", "d", "e", "f", "g"], "c"))},
+ {"id":8, "t8": (select array_binary_search(["a", "b", "c", "d", "e", "f", "g"], "a"))},
+ {"id":9, "t9": (select array_binary_search(["a", "b", "c", "d", "e", "f", "g"], "g"))},
+ {"id":10, "t10": (select array_binary_search([1,2,3,4,5,6,7,8], 3.0))},
+ {"id":11, "t11": (select array_binary_search([1,2,3,4,5,6,7,8], 8.0))},
+ {"id":12, "t12": (select array_binary_search([1,2,3,4,5,6,7,8], 2.5))},
+ {"id":13, "t13": (select array_binary_search("not-an-array", 3))},
+ {"id":14, "t14": (select array_binary_search([ ["a", "b"], ["b", "c"], ["c", "d"], ["d", "e"] ], ["a", "b"]))},
+ {"id":15, "t15": (select array_binary_search([ ["a", "b"], ["b", "c"], ["c", "d"], ["d", "e"] ], ["d", "e"]))},
+ {"id":16, "t16": (from openDs select array_binary_search(list_f, [9999,3]) order by id)},
+ {"id":17, "t17": (from openDs select array_binary_search(list_f, {"state": "OH", "country": "US"}) order by id)},
+ {"id":18, "t18": (from closedDs select array_binary_search(list_f, {"state": "OH", "country": "US"}) order by id)},
+ {"id":19, "t19": (select array_binary_search([{"id": 1, "age": 34}, {"id": 2, "age": 29}, {"id": 3, "age": 90}, {"id": 4, "age": 10}], {"id": 2, "age": 29} ))},
+ {"id":20, "t20": (select array_binary_search([0,0,1,1,1,2,3,3,3,3,4,5,6,6,6,7], 3))},
+ {"id":21, "t21": (select array_binary_search(["a", "b", "b", "b", "c", "d", "e", "e", "f", "f", "f", "g", "h", "i", "j"], "f"))},
+ {"id":22, "t22": (select array_binary_search( {{1, 2, 3, 4, 5}}, 3))}
+] as d
+
+SELECT VALUE d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.4.ddl.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.4.ddl.sqlpp
index 3c1a24d..3f8c8ec 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_binary_search/array_binary_search.4.ddl.sqlpp
@@ -16,20 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+drop dataverse TinySocial;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.1.ddl.sqlpp
new file mode 100644
index 0000000..601eb0c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.1.ddl.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse TinySocial if exists;
+create dataverse TinySocial;
+
+use TinySocial;
+
+create type addressType as closed {state: string, country: string, zip_code: int?};
+create type openType as {id: int};
+create type closedType as closed {id: int, list_f: [addressType]};
+
+create dataset openDs(openType) primary key id;
+create dataset closedDs(closedType) primary key id;
+
+create type TinySocial.TwitterUserType as
+{
+ `screen-name` : string,
+ lang : string,
+ friends_count : bigint,
+ statuses_count : bigint,
+ name : string,
+ followers_count : bigint
+};
+
+create type TinySocial.TweetMessageType as
+ closed {
+ tweetid : string,
+ user : TwitterUserType,
+ `sender-location` : point?,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string
+};
+
+create type t1 AS {
+
+};
+
+create type t2 AS {
+id: int,
+compType: t1
+};
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid hints (`CARDINALITY`=`100`);
+create dataset d1(t2) primary key id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.2.update.sqlpp
new file mode 100644
index 0000000..7e8e5ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.2.update.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use TinySocial;
+
+load dataset TweetMessages using localfs ((`path`=`asterix_nc1://data/tinysocial/twm.adm`),(`format`=`adm`));
+
+insert into d1([
+{"id":1, "compType":{"sth":33}},
+{"id":2, "compType":{"sth":44}, "followers":["John Green", "Emily Jones"]}
+]);
+
+insert into openDs([
+{"id": 1, "list_f": [ [1,2,1], [9999,3] ]},
+{"id": 2, "list_f": [ ["white","blue","magenta"], ["red", "black"] ]},
+{"id": 3, "list_f": [ 1 , 2 ]},
+{"id": 4, "list_f": [ {"state": "OH", "country": "US"} , {"state": "CA", "country": "US", "zip_code": 92863} ]},
+{"id": 5, "list_f": [ {"state": "OR", "country": "US", "zip_code": null} , {"state": "IL", "country": "US", "zip_code": 92863} ]},
+{"id": 6, "list_f": null},
+{"id": 7}
+]);
+
+insert into closedDs([
+{"id": 1, "list_f": [ {"state": "OH", "country": "US"} , {"state": "CA", "country": "US", "zip_code": 92863} ]},
+{"id": 2, "list_f": [ {"state": "OR", "country": "US", "zip_code": null} , {"state": "IL", "country": "US", "zip_code": 92863} ]}
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.3.query.sqlpp
new file mode 100644
index 0000000..8659ed4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.3.query.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use TinySocial;
+
+FROM [
+ {"id":1, "t1" : (select array_move([0,1,2,3,4,5], 1, 3))},
+ {"id":2, "t2" : (select array_move([0,1,2,3,4,5], -1, -3))},
+ {"id":3, "t3" : (select array_move(["a", "b", "c", "d", "e"], 0, 2))},
+ {"id":4, "t4" : (select array_move(["a", "b", "c", "d", "e"], -2, -4))},
+ {"id":5, "t5" : (select array_move(null, 0, 1))},
+ {"id":6, "t6" : (select array_move([], 0, 1))},
+ {"id":7, "t7" : (select array_move([1,2,3,4,5], null, 1))},
+ {"id":8, "t8" : (select array_move([1,2,3,4,5], 0, null))},
+ {"id":9, "t9": (from openDs select array_move(list_f, 0, 1) order by id)},
+ {"id":10, "t10": (from openDs select array_move(list_f, -1, -2) order by id)},
+ {"id":11, "t11": (from openDs select array_move(list_f, null, 1) order by id)},
+ {"id":12, "t12": (from openDs select array_move(list_f, missing, 1) order by id)},
+ {"id":13, "t13": (from closedDs select array_move(list_f, 0, 1) order by id)},
+ {"id":14, "t14": (from closedDs select array_move(list_f, -1, -2) order by id)},
+ {"id":15, "t15": (from closedDs select array_move(list_f, null, 1) order by id)},
+ {"id":16, "t16": (from closedDs select array_move(list_f, missing, 1) order by id)},
+ {"id":17, "t17": (select array_move("not-an-array", 0, 1))},
+ {"id":18, "t18": (select array_move([1,2,3,4,5], 7, 8))},
+ {"id":19, "t19": (select array_move([1,2,3,4,5], "not-an-int", 1))},
+ {"id":20, "t20": (select array_move([1,2,3,4,5], 0, "not-an-int"))},
+ {"id":21, "t21": (select array_move(missing, 0, 1))},
+ {"id":22, "t22": (select array_move([1,2,3,4,5], missing, 1))},
+ {"id":23, "t23": (select array_move([1,2,3,4,5], 0, missing))},
+ {"id":24, "t24": (select array_move([1,2,3,4,5], 4, 0))},
+ {"id":25, "t25": (select array_move([1,2,3,4,5], 3, 1))},
+ {"id":26, "t26": (select array_move( {{1,2,3,4,5}}, 0, 3 ))}
+] as d
+
+SELECT VALUE d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.4.ddl.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.4.ddl.sqlpp
index 3c1a24d..3f8c8ec 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_move/array_move.4.ddl.sqlpp
@@ -16,20 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+drop dataverse TinySocial;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.1.ddl.sqlpp
new file mode 100644
index 0000000..601eb0c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.1.ddl.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse TinySocial if exists;
+create dataverse TinySocial;
+
+use TinySocial;
+
+create type addressType as closed {state: string, country: string, zip_code: int?};
+create type openType as {id: int};
+create type closedType as closed {id: int, list_f: [addressType]};
+
+create dataset openDs(openType) primary key id;
+create dataset closedDs(closedType) primary key id;
+
+create type TinySocial.TwitterUserType as
+{
+ `screen-name` : string,
+ lang : string,
+ friends_count : bigint,
+ statuses_count : bigint,
+ name : string,
+ followers_count : bigint
+};
+
+create type TinySocial.TweetMessageType as
+ closed {
+ tweetid : string,
+ user : TwitterUserType,
+ `sender-location` : point?,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string
+};
+
+create type t1 AS {
+
+};
+
+create type t2 AS {
+id: int,
+compType: t1
+};
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid hints (`CARDINALITY`=`100`);
+create dataset d1(t2) primary key id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.2.update.sqlpp
new file mode 100644
index 0000000..7e8e5ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.2.update.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use TinySocial;
+
+load dataset TweetMessages using localfs ((`path`=`asterix_nc1://data/tinysocial/twm.adm`),(`format`=`adm`));
+
+insert into d1([
+{"id":1, "compType":{"sth":33}},
+{"id":2, "compType":{"sth":44}, "followers":["John Green", "Emily Jones"]}
+]);
+
+insert into openDs([
+{"id": 1, "list_f": [ [1,2,1], [9999,3] ]},
+{"id": 2, "list_f": [ ["white","blue","magenta"], ["red", "black"] ]},
+{"id": 3, "list_f": [ 1 , 2 ]},
+{"id": 4, "list_f": [ {"state": "OH", "country": "US"} , {"state": "CA", "country": "US", "zip_code": 92863} ]},
+{"id": 5, "list_f": [ {"state": "OR", "country": "US", "zip_code": null} , {"state": "IL", "country": "US", "zip_code": 92863} ]},
+{"id": 6, "list_f": null},
+{"id": 7}
+]);
+
+insert into closedDs([
+{"id": 1, "list_f": [ {"state": "OH", "country": "US"} , {"state": "CA", "country": "US", "zip_code": 92863} ]},
+{"id": 2, "list_f": [ {"state": "OR", "country": "US", "zip_code": null} , {"state": "IL", "country": "US", "zip_code": 92863} ]}
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.3.query.sqlpp
new file mode 100644
index 0000000..6689330
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.3.query.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use TinySocial;
+
+FROM [
+ {"id":1, "t1" : (select array_swap([0,1,2,3,4,5], 1, 3))},
+ {"id":2, "t2" : (select array_swap([0,1,2,3,4,5], -1, -3))},
+ {"id":3, "t3" : (select array_swap(["a", "b", "c", "d", "e"], 0, 2))},
+ {"id":4, "t4" : (select array_swap(["a", "b", "c", "d", "e"], -2, -4))},
+ {"id":5, "t5" : (select array_swap(null, 0, 1))},
+ {"id":6, "t6" : (select array_swap([], 0, 1))},
+ {"id":7, "t7" : (select array_swap([1,2,3,4,5], null, 1))},
+ {"id":8, "t8" : (select array_swap([1,2,3,4,5], 0, null))},
+ {"id":9, "t9": (from openDs select array_swap(list_f, 0, 1) order by id)},
+ {"id":10, "t10": (from openDs select array_swap(list_f, -1, -2) order by id)},
+ {"id":11, "t11": (from openDs select array_swap(list_f, null, 1) order by id)},
+ {"id":12, "t12": (from openDs select array_swap(list_f, missing, 1) order by id)},
+ {"id":13, "t13": (from closedDs select array_swap(list_f, 0, 1) order by id)},
+ {"id":14, "t14": (from closedDs select array_swap(list_f, -1, -2) order by id)},
+ {"id":15, "t15": (from closedDs select array_swap(list_f, null, 1) order by id)},
+ {"id":16, "t16": (from closedDs select array_swap(list_f, missing, 1) order by id)},
+ {"id":17, "t17" : (select array_swap("not-an-array", 0, 1))},
+ {"id":18, "t18" : (select array_swap([1,2,3,4,5], 7, 8))},
+ {"id":19, "t19" : (select array_swap([1,2,3,4,5], "not-an-int", 1))},
+ {"id":20, "t20" : (select array_swap([1,2,3,4,5], 0, "not-an-int"))},
+ {"id":21, "t21" : (select array_swap(missing, 0, 1))},
+ {"id":22, "t22" : (select array_swap([1,2,3,4,5], missing, 1))},
+ {"id":23, "t23" : (select array_swap([1,2,3,4,5], 0, missing))},
+ {"id":24, "t24": (select array_swap([1,2,3,4,5], 4, 0))},
+ {"id":25, "t25": (select array_swap([1,2,3,4,5], 3, 1))},
+ {"id":26, "t26": (select array_swap({{1,2,3,4,5}}, 1, 3))}
+] as d
+
+SELECT VALUE d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.4.ddl.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.4.ddl.sqlpp
index 3c1a24d..3f8c8ec 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_swap/array_swap.4.ddl.sqlpp
@@ -16,20 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+drop dataverse TinySocial;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml
index 47eb4395..9a5e17e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml
@@ -20,44 +20,52 @@
<test-case FilePath="async-deferred">
<compilation-unit name="async-failed">
<output-dir compare="Clean-JSON">async-failed</output-dir>
+ <parameter name="profile" value="timings" type="string"/>
<expected-error>Injected failure in inject-failure</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="async-deferred">
<compilation-unit name="async-compilation-failed">
+ <parameter name="profile" value="timings" type="string"/>
<output-dir compare="Clean-JSON">async-compilation-failed</output-dir>
<expected-error>Cannot find dataset gargel</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="async-deferred">
<compilation-unit name="deferred">
+ <parameter name="profile" value="timings" type="string"/>
<output-dir compare="Clean-JSON">deferred</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="async-deferred">
<compilation-unit name="async">
+ <parameter name="profile" value="timings" type="string"/>
<output-dir compare="Clean-JSON">async</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="async-deferred">
<compilation-unit name="async-repeated">
+ <parameter name="profile" value="timings" type="string"/>
<output-dir compare="Clean-JSON">async-repeated</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="async-deferred">
<compilation-unit name="async-running">
+ <parameter name="profile" value="timings" type="string"/>
<output-dir compare="Clean-JSON">async-running</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="async-deferred">
<compilation-unit name="async-exhausted-result">
<output-dir compare="Clean-JSON">async-exhausted-result</output-dir>
+ <parameter name="profile" value="timings" type="string"/>
<expected-error>Premature end of chunk</expected-error> <!--TODO:REVISIT -->
<source-location>false</source-location>
</compilation-unit>
</test-case>
<test-case FilePath="async-deferred">
<compilation-unit name="async-json">
+ <parameter name="profile" value="timings" type="string"/>
<output-dir compare="Clean-JSON">async-json</output-dir>
</compilation-unit>
</test-case>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.query.sqlpp
index 93b09d0..e8f7963 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.query.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
/*
-* Description : Pushdown "p.entities.urls"
+* Description : Pushdown "p.entities.urls[*].display_url"
* Expected Res : Success
* Date : July 23th 2021
*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
index 8532fe9..77fc7c5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
@@ -124,6 +124,10 @@
<compilation-unit name="object_concat">
<output-dir compare="Text">object_concat</output-dir>
<expected-warn>ASX0013: Duplicate field name 'v'</expected-warn>
+ <expected-warn>ASX0013: Duplicate field name 'f'</expected-warn>
+ <expected-warn>ASX0013: Duplicate field name 'id'</expected-warn>
+ <expected-warn>ASX0013: Duplicate field name 'id'</expected-warn>
+ <expected-warn>ASX0013: Duplicate field name 'dup'</expected-warn>
</compilation-unit>
</test-case>
<test-case FilePath="objects">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.5.ddl.sqlpp
new file mode 100644
index 0000000..2eefbe8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.5.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE TestDataverse IF EXISTS;
+CREATE DATAVERSE TestDataverse;
+USE TestDataverse;
+
+CREATE TYPE UsersBaseTypeClosed AS CLOSED { _id: int };
+CREATE TYPE UsersBaseTypeOpen AS { _id: int };
+CREATE TYPE UsersFriendsTypeClosed AS CLOSED {
+ _id: int,
+ best_friend: UsersBaseTypeClosed,
+ friends: [UsersBaseTypeClosed]
+};
+CREATE TYPE UsersFriendsTypeOpen AS { _id: int };
+
+CREATE DATASET UsersClosed (UsersFriendsTypeClosed) PRIMARY KEY _id;
+CREATE DATASET UsersOpen (UsersFriendsTypeOpen) PRIMARY KEY _id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.6.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.6.update.sqlpp
new file mode 100644
index 0000000..bda29eb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.6.update.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE TestDataverse;
+
+INSERT INTO UsersClosed [
+ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [] },
+ { "_id": 2, "best_friend": { "_id": 1 }, "friends": [{ "_id": 1 }] },
+ { "_id": 3, "best_friend": { "_id": 2 }, "friends": [{ "_id": 1 }, { "_id": 2 }] }
+];
+INSERT INTO UsersOpen [
+ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [] },
+ { "_id": 5, "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [{ "_id": 4 }] },
+ { "_id": 6, "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [{ "_id": 4 }, { "_id": 5 }] }
+];
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.7.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.7.query.sqlpp
new file mode 100644
index 0000000..71f6974
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_add/object_add.7.query.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// For OBJECT_ADD.
+USE TestDataverse;
+LET openUValues = (
+ FROM UsersOpen U2
+ WHERE U2._id = 4
+ SELECT VALUE U2
+ ),
+ closedUValues = (
+ FROM UsersClosed U1
+ WHERE U1._id = 1
+ SELECT VALUE U1
+ )
+SELECT VALUE {
+ // New constant field into closed and open record.
+ "t1c": ( FROM UsersClosed U SELECT VALUE OBJECT_ADD(U, "name", "John") ORDER BY U._id ),
+ "t1o": ( FROM UsersOpen U SELECT VALUE OBJECT_ADD(U, "name", "John") ORDER BY U._id ),
+
+ // New record field from open record into closed and open record.
+ "t2c": ( FROM UsersClosed U SELECT VALUE OBJECT_ADD(U, "newFriend", openUValues[0]) ORDER BY U._id ) ,
+ "t2o": ( FROM UsersOpen U SELECT VALUE OBJECT_ADD(U, "newFriend", openUValues[0]) ORDER BY U._id ) ,
+
+ // New record field from closed record into closed and open record.
+ "t3c": ( FROM UsersClosed U SELECT VALUE OBJECT_ADD(U, "newFriend", closedUValues[0]) ORDER BY U._id ),
+ "t3o": ( FROM UsersOpen U SELECT VALUE OBJECT_ADD(U, "newFriend", closedUValues[0]) ORDER BY U._id ),
+
+ // New list field from open dataset into closed and open record.
+ "t4c": ( FROM UsersClosed U SELECT VALUE OBJECT_ADD(U, "newFriends", openUValues) ORDER BY U._id ),
+ "t4o": ( FROM UsersOpen U SELECT VALUE OBJECT_ADD(U, "newFriends", openUValues) ORDER BY U._id ),
+
+ // New list field from closed dataset into closed and open record.
+ "t5c": ( FROM UsersClosed U SELECT VALUE OBJECT_ADD(U, "newFriends", closedUValues) ORDER BY U._id ),
+ "t5o": ( FROM UsersOpen U SELECT VALUE OBJECT_ADD(U, "newFriends", closedUValues) ORDER BY U._id )
+};
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_concat/object_concat.5.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_concat/object_concat.5.query.sqlpp
index 3c1a24d..1b61e66 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_concat/object_concat.5.query.sqlpp
@@ -16,20 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+// test that object_concat() issues a warning when encountering a duplicate field
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// requesttype=application/json
+// param max-warnings:json=10
+
+SET `import-private-functions` `true`;
+WITH t AS ([{"id": 1, "f": 3}, {"id": 2, "f": 4}])
+SELECT OBJECT_CONCAT_STRICT(t) AS res;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_concat/object_concat.6.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_concat/object_concat.6.query.sqlpp
index 3c1a24d..8ef2ebbb 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_concat/object_concat.6.query.sqlpp
@@ -16,20 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+// requesttype=application/json
+// param max-warnings:json=10
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+WITH a AS (SELECT VALUE x FROM [{"id": 1, "a1": 3, "dup": 0}] AS x),
+b AS (SELECT VALUE y FROM [{"id": 1, "b1": 3, "dup": 5}] AS y)
+FROM a, b
+WHERE a.id = b.id
+SELECT a.*, b.*
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.5.ddl.sqlpp
new file mode 100644
index 0000000..2eefbe8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.5.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE TestDataverse IF EXISTS;
+CREATE DATAVERSE TestDataverse;
+USE TestDataverse;
+
+CREATE TYPE UsersBaseTypeClosed AS CLOSED { _id: int };
+CREATE TYPE UsersBaseTypeOpen AS { _id: int };
+CREATE TYPE UsersFriendsTypeClosed AS CLOSED {
+ _id: int,
+ best_friend: UsersBaseTypeClosed,
+ friends: [UsersBaseTypeClosed]
+};
+CREATE TYPE UsersFriendsTypeOpen AS { _id: int };
+
+CREATE DATASET UsersClosed (UsersFriendsTypeClosed) PRIMARY KEY _id;
+CREATE DATASET UsersOpen (UsersFriendsTypeOpen) PRIMARY KEY _id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.6.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.6.update.sqlpp
new file mode 100644
index 0000000..bda29eb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.6.update.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE TestDataverse;
+
+INSERT INTO UsersClosed [
+ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [] },
+ { "_id": 2, "best_friend": { "_id": 1 }, "friends": [{ "_id": 1 }] },
+ { "_id": 3, "best_friend": { "_id": 2 }, "friends": [{ "_id": 1 }, { "_id": 2 }] }
+];
+INSERT INTO UsersOpen [
+ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [] },
+ { "_id": 5, "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [{ "_id": 4 }] },
+ { "_id": 6, "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [{ "_id": 4 }, { "_id": 5 }] }
+];
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.7.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.7.query.sqlpp
new file mode 100644
index 0000000..52943a0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_put/object_put.7.query.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// For OBJECT_PUT.
+USE TestDataverse;
+LET openUValues = (
+ FROM UsersOpen U2
+ WHERE U2._id = 4
+ SELECT VALUE U2
+ ),
+ closedUValues = (
+ FROM UsersClosed U1
+ WHERE U1._id = 1
+ SELECT VALUE U1
+ )
+SELECT VALUE {
+ // New constant field into closed and open record.
+ "t1c": ( FROM UsersClosed U SELECT VALUE OBJECT_PUT(U, "name", "John") ORDER BY U._id ),
+ "t1o": ( FROM UsersOpen U SELECT VALUE OBJECT_PUT(U, "name", "John") ORDER BY U._id ),
+
+ // New record field from open record into closed and open record.
+ "t2c": ( FROM UsersClosed U SELECT VALUE OBJECT_PUT(U, "newFriend", openUValues[0]) ORDER BY U._id ) ,
+ "t2o": ( FROM UsersOpen U SELECT VALUE OBJECT_PUT(U, "newFriend", openUValues[0]) ORDER BY U._id ) ,
+
+ // New record field from closed record into closed and open record.
+ "t3c": ( FROM UsersClosed U SELECT VALUE OBJECT_PUT(U, "newFriend", closedUValues[0]) ORDER BY U._id ),
+ "t3o": ( FROM UsersOpen U SELECT VALUE OBJECT_PUT(U, "newFriend", closedUValues[0]) ORDER BY U._id ),
+
+ // New list field from open dataset into closed and open record.
+ "t4c": ( FROM UsersClosed U SELECT VALUE OBJECT_PUT(U, "newFriends", openUValues) ORDER BY U._id ),
+ "t4o": ( FROM UsersOpen U SELECT VALUE OBJECT_PUT(U, "newFriends", openUValues) ORDER BY U._id ),
+
+ // New list field from closed dataset into closed and open record.
+ "t5c": ( FROM UsersClosed U SELECT VALUE OBJECT_PUT(U, "newFriends", closedUValues) ORDER BY U._id ),
+ "t5o": ( FROM UsersOpen U SELECT VALUE OBJECT_PUT(U, "newFriends", closedUValues) ORDER BY U._id ),
+
+ // Old field of open type into closed record (we should overwrite).
+ "t6c": ( FROM UsersClosed U
+ LET newFriends = [
+ { "_id": 8 }, { "_id": 9 }
+ ]
+ SELECT VALUE OBJECT_PUT(U, "friends", newFriends)
+ ORDER BY U._id )
+};
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.1.ddl.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.1.ddl.sqlpp
index 3c1a24d..8fc7172 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.1.ddl.sqlpp
@@ -16,20 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+DROP DATAVERSE ComplexExclude IF EXISTS;
+CREATE DATAVERSE ComplexExclude;
+USE ComplexExclude;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
+
+CREATE VIEW UsersWithoutPII AS
+ FROM ComplexExclude.Users U
+ SELECT U.* EXCLUDE phones, address, ssn;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.10.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.10.query.sqlpp
index 3c1a24d..67d91fb 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.10.query.sqlpp
@@ -16,20 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with RIGHT JOIN and EXCLUDE.
+FROM ComplexExclude.Users U2
+RIGHT JOIN ComplexExclude.Users U1
+ON U1.best_friend = U2.user_id
+SELECT * EXCLUDE U1.address, U2.address
+ORDER BY U1.user_id, U2.user_id;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.11.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.11.query.sqlpp
index 3c1a24d..099c6c5 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.11.query.sqlpp
@@ -16,20 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with DISTINCT (applies after EXCLUDE) and normal WHERE.
+FROM ComplexExclude.Users U
+WHERE U.best_friend = 1
+SELECT DISTINCT U.* EXCLUDE address, favorite_color, user_id;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.12.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.12.query.sqlpp
index 3c1a24d..6b5ac14 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.12.query.sqlpp
@@ -16,20 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query using view with EXCLUDE + EXCLUDE on top.
+FROM ComplexExclude.UsersWithoutPII U
+SELECT U.* EXCLUDE favorite_color
+ORDER BY U.user_id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.13.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.13.query.sqlpp
index 3c1a24d..46bac5d 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.13.query.sqlpp
@@ -16,20 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query using EXCLUDE as a field-name.
+// (We disallow EXCLUDE as a field-name if not preceded with an 'AS').
+WITH A AS [ { "a": 1, "b": 2, "EXCLUDE": 3 } ]
+FROM A
+SELECT A AS EXCLUDE
+UNION ALL
+FROM A
+SELECT A.EXCLUDE
+UNION ALL
+FROM A
+SELECT A EXCLUDE A.a
+ORDER BY EXCLUDE;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.14.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.14.query.sqlpp
new file mode 100644
index 0000000..20975a4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.14.query.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Query using all clauses + EXCLUDE.
+WITH otherUsers AS ( FROM ComplexExclude.Users U SELECT VALUE U )
+FROM ComplexExclude.Users U1
+JOIN otherUsers U2
+ON U2.user_id = U1.best_friend
+LEFT UNNEST U1.phones U1P
+LET bestFriend = U2
+WHERE U1.user_id = U2.best_friend
+GROUP BY U1
+GROUP AS G
+LET bestFriends = ( FROM G SELECT VALUE bestFriend )
+HAVING COUNT(*) > 0
+SELECT bestFriends, U1.* EXCLUDE address, phones
+ORDER BY U1.user_id
+LIMIT 10;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.2.update.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.2.update.sqlpp
index 3c1a24d..4c770f6 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.2.update.sqlpp
@@ -16,20 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+USE ComplexExclude;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+INSERT INTO Users [
+ { "user_id": 1, "best_friend": 2, "phones": [ { "kind": "MOBILE", "number": "222-222-2222" } ] },
+ { "user_id": 2, "best_friend": 1, "address": { "zip_code": "99929", "street": "2341 Apple Street" } },
+ { "user_id": 3, "best_friend": 1, "address": { "zip_code": "99929", "street": "2341 Apple Street" }, "favorite_color": "Green" },
+ { "user_id": 4, "best_friend": null }
+];
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.3.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.3.query.sqlpp
index 3c1a24d..bc01577 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.3.query.sqlpp
@@ -16,20 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query using view with EXCLUDE.
+FROM ComplexExclude.UsersWithoutPII U
+SELECT U.*
+ORDER BY U.user_id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.4.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.4.query.sqlpp
index 3c1a24d..6aa8a33 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.4.query.sqlpp
@@ -16,20 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with extraneous EXCLUDE terms ("address.zip_code" is unnecessary).
+FROM ComplexExclude.Users U
+SELECT U.* EXCLUDE address, address.zip_code
+ORDER BY U.user_id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.5.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.5.query.sqlpp
index 3c1a24d..8ef1df6 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.5.query.sqlpp
@@ -16,20 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with EXCLUDE in subquery.
+FROM ComplexExclude.Users U1
+WHERE U1.best_friend = 1
+SELECT VALUE ( FROM ComplexExclude.Users U2
+ WHERE U2.user_id = U1.user_id
+ SELECT U2.* EXCLUDE address )
+ORDER BY U1.user_id;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.6.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.6.query.sqlpp
index 3c1a24d..8e0b660 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.6.query.sqlpp
@@ -16,20 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with EXCLUDE on the projections w/o star.
+FROM ComplexExclude.Users U
+SELECT U.user_id, U.best_friend, U.address EXCLUDE best_friend, address
+ORDER BY U.user_id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.7.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.7.query.sqlpp
index 3c1a24d..ba6588e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.7.query.sqlpp
@@ -16,20 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with GROUP-BY and EXCLUDE.
+FROM ComplexExclude.Users U
+GROUP BY U.best_friend
+GROUP AS G
+SELECT * EXCLUDE G
+ORDER BY U.best_friend NULLS LAST;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.8.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.8.query.sqlpp
index 3c1a24d..437c7af 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.8.query.sqlpp
@@ -16,20 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with CTE, EXCLUDE, and explicit JOIN.
+LET SpecialUsers = (
+ FROM ComplexExclude.Users U
+ WHERE U.user_id = 1
+ SELECT VALUE U
+ )
+FROM ComplexExclude.Users U
+INNER JOIN SpecialUsers S
+ON U.user_id = S.user_id
+SELECT U.* EXCLUDE phones;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.9.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.9.query.sqlpp
index 3c1a24d..ff507a3 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/complex-exclude/complex-exclude.9.query.sqlpp
@@ -16,20 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// Query with LET after the FROM clause.
+FROM ComplexExclude.Users U
+LET miscInfo = { "24as": "23412", "address": "2341 Orange Street" }
+SELECT * EXCLUDE U.address, miscInfo.address, U.phones
+ORDER BY U.user_id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.1.ddl.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.1.ddl.sqlpp
index 3c1a24d..7c54e82 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.1.ddl.sqlpp
@@ -16,20 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+DROP DATAVERSE ExcludeNegative IF EXISTS;
+CREATE DATAVERSE ExcludeNegative;
+USE ExcludeNegative;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+CREATE TYPE UsersType AS { user_id: bigint };
+CREATE DATASET Users (UsersType) PRIMARY KEY user_id;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.2.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.2.query.sqlpp
index 3c1a24d..8e8c021 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.2.query.sqlpp
@@ -16,20 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// EXCLUDE cannot be used as an alias without 'AS'.
+FROM NegativeExclude.Users U
+SELECT U.user_id EXCLUDE;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.3.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.3.query.sqlpp
index 3c1a24d..eee1023 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/exclude-negative/exclude-negative.3.query.sqlpp
@@ -16,20 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+// EXCLUDE cannot be used with SELECT VALUE.
+FROM NegativeExclude.Users U
+SELECT VALUE U EXCLUDE user_id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.1.ddl.sqlpp
new file mode 100644
index 0000000..3cb07d0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.1.ddl.sqlpp
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE TinySocial IF EXISTS;
+CREATE DATAVERSE TinySocial;
+USE TinySocial;
+
+CREATE TYPE TwitterUserType AS {
+ `screen-name` : string,
+ lang : string,
+ friends_count : bigint,
+ statuses_count : bigint
+};
+
+CREATE TYPE TweetMessageType AS {
+ tweetid : string,
+ user : TwitterUserType,
+ `sender-location` : point?,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string
+};
+
+CREATE DATASET TwitterUsers(TwitterUserType) PRIMARY KEY `screen-name`;
+CREATE DATASET TweetMessages(TweetMessageType) PRIMARY KEY tweetid;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.2.update.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.2.update.sqlpp
index 3c1a24d..a60ef4d 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.2.update.sqlpp
@@ -16,20 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+USE TinySocial;
+LOAD DATASET TwitterUsers USING localfs ((`path`=`asterix_nc1://data/tinysocial/twu.adm`),(`format`=`adm`));
+LOAD DATASET TweetMessages USING localfs ((`path`=`asterix_nc1://data/tinysocial/twm.adm`),(`format`=`adm`));
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.3.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.3.query.sqlpp
index 3c1a24d..4d7489a 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.3.query.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+FROM TinySocial.TweetMessages TM
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+ // SELECT-var.*.
+SELECT TM.*
+
+ // Nested field that does not exist.
+EXCLUDE user.does_not_exist,
+ // Nested field that does exist.
+ user.lang,
+ // Field with quotes.
+ `send-time`,
+ // Non-nested field that does not exist.
+ does_not_exist,
+ // Non-nested field that exists.
+ tweetid
+
+ORDER BY TM.tweetid;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.4.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.4.query.sqlpp
index 3c1a24d..06c6df0 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.4.query.sqlpp
@@ -16,20 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+FROM TinySocial.TweetMessages TM,
+ TinySocial.TwitterUsers TU
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+WHERE TM.user.`screen-name` = TU.`screen-name`
+
+ // SELECT * returns { "TM": ..., "TU": ... }
+SELECT *
+
+ // Identifier that exists in scope.
+EXCLUDE TU,
+ // Nested identifier that exists in scope.
+ TM.user.`screen-name`,
+ // Identifier that does not exist in scope.
+ TI
+
+ORDER BY TM.tweetid;
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.5.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.5.query.sqlpp
index 3c1a24d..d41b3a7 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.5.query.sqlpp
@@ -16,20 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+FROM TinySocial.TweetMessages TM1
+SELECT TM1.*
+EXCLUDE user.does_not_exist,
+ user.lang,
+ `send-time`,
+ does_not_exist
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+UNION ALL
+
+FROM TinySocial.TweetMessages TM2
+SELECT TM2.*
+EXCLUDE user.lang
+
+ORDER BY tweetid, `send-time` NULLS FIRST;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.6.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.6.query.sqlpp
index 3c1a24d..e5453ef 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-exclude/tiny-social/tiny-social.6.query.sqlpp
@@ -16,20 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+FROM TinySocial.TweetMessages TM
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+ // SELECT-* (single-variable rule case).
+SELECT *
+
+ // Nested field that exists, anchored by TM.
+EXCLUDE TM.user.lang,
+ // Field with quotes (single-variable rule applies).
+ `send-time`,
+ // Non-nested field that exists (single-variable rule applies).
+ tweetid
+
+ORDER BY tweetid;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.1.ddl.sqlpp
new file mode 100644
index 0000000..2a616ea
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.1.ddl.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE TinySocial IF EXISTS;
+CREATE DATAVERSE TinySocial;
+
+USE TinySocial;
+
+CREATE TYPE EmploymentType AS CLOSED
+{
+ `organization-name` : string,
+ `start-date` : date,
+ `end-date` : date?
+};
+
+CREATE TYPE FacebookUserType AS CLOSED {
+ id : bigint,
+ alias : string,
+ name : string,
+ `user-since` : datetime,
+ `friend-ids` : {{bigint}},
+ employment : [EmploymentType]
+};
+
+CREATE TYPE FacebookMessageType AS CLOSED {
+ `message-id` : bigint,
+ `author-id` : bigint,
+ `in-response-to` : bigint?,
+ `sender-location` : point?,
+ message : string
+};
+
+CREATE DATASET FacebookUsers(FacebookUserType) PRIMARY KEY id;
+
+CREATE DATASET FacebookMessages(FacebookMessageType) PRIMARY KEY `message-id`;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.2.update.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.2.update.sqlpp
index 3c1a24d..3d785ad 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.2.update.sqlpp
@@ -16,20 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+USE TinySocial;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+LOAD DATASET FacebookUsers using localfs (("path"="asterix_nc1://data/tinysocial/fbu.adm"),("format"="adm"));
+
+LOAD DATASET FacebookMessages using localfs (("path"="asterix_nc1://data/tinysocial/fbm.adm"),("format"="adm"));
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.3.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.3.query.sqlpp
index 3c1a24d..003ec26 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.3.query.sqlpp
@@ -16,20 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+USE TinySocial;
+-- Disabled for a simpler plan
+SET `compiler.sort.parallel` "false";
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+
+ SELECT "message1" ds, s.`message-id` id, object_remove(s, "in-response-to") no_in_response_to
+ FROM FacebookMessages AS s
+UNION ALL
+ SELECT "user" ds, t.id id, t user
+ FROM FacebookUsers t
+UNION ALL
+ SELECT "message2" ds, s.`message-id` id, object_remove(s, "author-id") no_author_id
+ FROM FacebookMessages s
+ORDER BY id, ds;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.4.query.sqlpp
similarity index 62%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.4.query.sqlpp
index 3c1a24d..cc63b57 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_type_cast/union_type_cast.4.query.sqlpp
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+USE TinySocial;
+-- Disabled for a simpler plan
+SET `compiler.sort.parallel` "false";
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
-}
+-- To prevent the plan from changing
+SET `compiler.parallelism` "0";
+
+EXPLAIN
+ SELECT s.`message-id` id, object_remove(s, "in-response-to") no_in_response_to
+ FROM FacebookMessages AS s
+UNION ALL
+ SELECT t.id id, t user
+ FROM FacebookUsers t
+UNION ALL
+ SELECT s.`message-id` id, object_remove(s, "author-id") no_author_id
+ FROM FacebookMessages s
+ORDER BY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index dd6ac17..4e6a68e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -10,7 +10,9 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"compiler\.arrayindex" : true,
+ "compiler.batch.lookup" : false,
"compiler.cbo" : false,
+ "compiler.column.filter" : false,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 9103663..d025d7a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -10,7 +10,9 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"compiler\.arrayindex" : true,
+ "compiler.batch.lookup" : false,
"compiler.cbo" : false,
+ "compiler.column.filter" : false,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 7ae9886..7f2dcb1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -10,7 +10,9 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"compiler\.arrayindex" : true,
+ "compiler.batch.lookup" : false,
"compiler.cbo" : false,
+ "compiler.column.filter" : false,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.regexjson
index ad95b7b..6fa99e2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.regexjson
@@ -6,6 +6,7 @@
"metrics": {
"elapsedTime": "R{.*}",
"executionTime": "R{.*}",
+ "compileTime": "R{.*}",
"resultCount": 0,
"resultSize": 0,
"processedObjects": 0,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_binary_search/array_binary_search.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_binary_search/array_binary_search.3.adm
new file mode 100644
index 0000000..ae6f41f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_binary_search/array_binary_search.3.adm
@@ -0,0 +1,23 @@
+{ "id": 1, "t1": [ { "$1": null } ] }
+{ "id": 2, "t2": [ { } ] }
+{ "id": 3, "t3": [ { "$3": -1 } ] }
+{ "id": 4, "t4": [ { "$4": 3 } ] }
+{ "id": 5, "t5": [ { "$5": 0 } ] }
+{ "id": 6, "t6": [ { "$6": 6 } ] }
+{ "id": 7, "t7": [ { "$7": 2 } ] }
+{ "id": 8, "t8": [ { "$8": 0 } ] }
+{ "id": 9, "t9": [ { "$9": 6 } ] }
+{ "id": 10, "t10": [ { "$10": 2 } ] }
+{ "id": 11, "t11": [ { "$11": 7 } ] }
+{ "id": 12, "t12": [ { "$12": -1 } ] }
+{ "id": 13, "t13": [ { "$13": null } ] }
+{ "id": 14, "t14": [ { "$14": 0 } ] }
+{ "id": 15, "t15": [ { "$15": 3 } ] }
+{ "id": 16, "t16": [ { "$16": 1 }, { "$16": -1 }, { "$16": -1 }, { "$16": -1 }, { "$16": -1 }, { "$16": null }, { } ] }
+{ "id": 17, "t17": [ { "$17": -1 }, { "$17": -1 }, { "$17": -1 }, { "$17": 0 }, { "$17": -1 }, { "$17": null }, { } ] }
+{ "id": 18, "t18": [ { "$18": 0 }, { "$18": -1 } ] }
+{ "id": 19, "t19": [ { "$19": 1 } ] }
+{ "id": 20, "t20": [ { "$20": 6 } ] }
+{ "id": 21, "t21": [ { "$21": 8 } ] }
+{ "id": 22, "t22": [ { "$22": null } ] }
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_move/array_move.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_move/array_move.3.adm
new file mode 100644
index 0000000..f8057c6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_move/array_move.3.adm
@@ -0,0 +1,26 @@
+{ "id": 1, "t1": [ { "$1": [ 0, 2, 3, 1, 4, 5 ] } ] }
+{ "id": 2, "t2": [ { "$2": [ 0, 1, 2, 5, 3, 4 ] } ] }
+{ "id": 3, "t3": [ { "$3": [ "b", "c", "a", "d", "e" ] } ] }
+{ "id": 4, "t4": [ { "$4": [ "a", "d", "b", "c", "e" ] } ] }
+{ "id": 5, "t5": [ { "$5": null } ] }
+{ "id": 6, "t6": [ { "$6": null } ] }
+{ "id": 7, "t7": [ { "$7": null } ] }
+{ "id": 8, "t8": [ { "$8": null } ] }
+{ "id": 9, "t9": [ { "$9": [ [ 9999, 3 ], [ 1, 2, 1 ] ] }, { "$9": [ [ "red", "black" ], [ "white", "blue", "magenta" ] ] }, { "$9": [ 2, 1 ] }, { "$9": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$9": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] }, { "$9": null }, { } ] }
+{ "id": 10, "t10": [ { "$10": [ [ 9999, 3 ], [ 1, 2, 1 ] ] }, { "$10": [ [ "red", "black" ], [ "white", "blue", "magenta" ] ] }, { "$10": [ 2, 1 ] }, { "$10": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$10": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] }, { "$10": null }, { } ] }
+{ "id": 11, "t11": [ { "$11": null }, { "$11": null }, { "$11": null }, { "$11": null }, { "$11": null }, { "$11": null }, { } ] }
+{ "id": 12, "t12": [ { }, { }, { }, { }, { }, { }, { } ] }
+{ "id": 13, "t13": [ { "$13": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$13": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] } ] }
+{ "id": 14, "t14": [ { "$14": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$14": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] } ] }
+{ "id": 15, "t15": [ { "$15": null }, { "$15": null } ] }
+{ "id": 16, "t16": [ { }, { } ] }
+{ "id": 17, "t17": [ { "$17": null } ] }
+{ "id": 18, "t18": [ { "$18": null } ] }
+{ "id": 19, "t19": [ { "$19": null } ] }
+{ "id": 20, "t20": [ { "$20": null } ] }
+{ "id": 21, "t21": [ { } ] }
+{ "id": 22, "t22": [ { } ] }
+{ "id": 23, "t23": [ { } ] }
+{ "id": 24, "t24": [ { "$24": [ 5, 1, 2, 3, 4 ] } ] }
+{ "id": 25, "t25": [ { "$25": [ 1, 4, 2, 3, 5 ] } ] }
+{ "id": 26, "t26": [ { "$26": null } ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_swap/array_swap.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_swap/array_swap.3.adm
new file mode 100644
index 0000000..97ccdc4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_swap/array_swap.3.adm
@@ -0,0 +1,26 @@
+{ "id": 1, "t1": [ { "$1": [ 0, 3, 2, 1, 4, 5 ] } ] }
+{ "id": 2, "t2": [ { "$2": [ 0, 1, 2, 5, 4, 3 ] } ] }
+{ "id": 3, "t3": [ { "$3": [ "c", "b", "a", "d", "e" ] } ] }
+{ "id": 4, "t4": [ { "$4": [ "a", "d", "c", "b", "e" ] } ] }
+{ "id": 5, "t5": [ { "$5": null } ] }
+{ "id": 6, "t6": [ { "$6": null } ] }
+{ "id": 7, "t7": [ { "$7": null } ] }
+{ "id": 8, "t8": [ { "$8": null } ] }
+{ "id": 9, "t9": [ { "$9": [ [ 9999, 3 ], [ 1, 2, 1 ] ] }, { "$9": [ [ "red", "black" ], [ "white", "blue", "magenta" ] ] }, { "$9": [ 2, 1 ] }, { "$9": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$9": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] }, { "$9": null }, { } ] }
+{ "id": 10, "t10": [ { "$10": [ [ 9999, 3 ], [ 1, 2, 1 ] ] }, { "$10": [ [ "red", "black" ], [ "white", "blue", "magenta" ] ] }, { "$10": [ 2, 1 ] }, { "$10": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$10": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] }, { "$10": null }, { } ] }
+{ "id": 11, "t11": [ { "$11": null }, { "$11": null }, { "$11": null }, { "$11": null }, { "$11": null }, { "$11": null }, { } ] }
+{ "id": 12, "t12": [ { }, { }, { }, { }, { }, { }, { } ] }
+{ "id": 13, "t13": [ { "$13": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$13": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] } ] }
+{ "id": 14, "t14": [ { "$14": [ { "state": "CA", "country": "US", "zip_code": 92863 }, { "state": "OH", "country": "US" } ] }, { "$14": [ { "state": "IL", "country": "US", "zip_code": 92863 }, { "state": "OR", "country": "US", "zip_code": null } ] } ] }
+{ "id": 15, "t15": [ { "$15": null }, { "$15": null } ] }
+{ "id": 16, "t16": [ { }, { } ] }
+{ "id": 17, "t17": [ { "$17": null } ] }
+{ "id": 18, "t18": [ { "$18": null } ] }
+{ "id": 19, "t19": [ { "$19": null } ] }
+{ "id": 20, "t20": [ { "$20": null } ] }
+{ "id": 21, "t21": [ { } ] }
+{ "id": 22, "t22": [ { } ] }
+{ "id": 23, "t23": [ { } ] }
+{ "id": 24, "t24": [ { "$24": [ 5, 2, 3, 4, 1 ] } ] }
+{ "id": 25, "t25": [ { "$25": [ 1, 4, 3, 2, 5 ] } ] }
+{ "id": 26, "t26": [ { "$26": null } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-exhausted-result/async-exhausted-result.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-exhausted-result/async-exhausted-result.3.regexjson
index df5177b..7fe2d93 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-exhausted-result/async-exhausted-result.3.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-exhausted-result/async-exhausted-result.3.regexjson
@@ -11,5 +11,6 @@
{ "i": 9, "i2": 81 },
{ "i": 10, "i2": 100 }
],
- "metrics": "R{.*}"
+ "metrics": "R{.*}",
+ "profile": "R{.*}"
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.3.regexjson
index df5177b..7fe2d93 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.3.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.3.regexjson
@@ -11,5 +11,6 @@
{ "i": 9, "i2": 81 },
{ "i": 10, "i2": 100 }
],
- "metrics": "R{.*}"
+ "metrics": "R{.*}",
+ "profile": "R{.*}"
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.4.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.4.regexjson
index df5177b..7fe2d93 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.4.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-repeated/async-repeated.4.regexjson
@@ -11,5 +11,6 @@
{ "i": 9, "i2": 81 },
{ "i": 10, "i2": 100 }
],
- "metrics": "R{.*}"
+ "metrics": "R{.*}",
+ "profile": "R{.*}"
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.4.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.4.regexjson
index a2a5f0a8..3e0193d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.4.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.4.regexjson
@@ -2,5 +2,6 @@
"results": [
"result"
],
- "metrics": "R{.*}"
+ "metrics": "R{.*}",
+ "profile": "R{.*}"
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.3.regexjson
index df5177b..7fe2d93 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.3.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.3.regexjson
@@ -11,5 +11,6 @@
{ "i": 9, "i2": 81 },
{ "i": 10, "i2": 100 }
],
- "metrics": "R{.*}"
+ "metrics": "R{.*}",
+ "profile": "R{.*}"
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/deferred/deferred.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/deferred/deferred.2.regexjson
index df5177b..7fe2d93 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/deferred/deferred.2.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/deferred/deferred.2.regexjson
@@ -11,5 +11,6 @@
{ "i": 9, "i2": 81 },
{ "i": 10, "i2": 100 }
],
- "metrics": "R{.*}"
+ "metrics": "R{.*}",
+ "profile": "R{.*}"
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.plan
index 7a7a890..42508a1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.plan
@@ -18,7 +18,7 @@
-- ASSIGN |PARTITIONED|
exchange [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset1 condition (gt(sql-count($$p.getField("entities").getField("urls")), 10)) limit 10 project ({entities:{urls:any}}) [cardinality: 1000000.0, op-cost: 1000000.0, total-cost: 1000000.0]
+ data-scan []<-[$$p] <- test.ParquetDataset1 condition (gt(sql-count($$p.getField("entities").getField("urls")), 10)) limit 10 project ({entities:{urls:[{display_url:any}]}}) [cardinality: 1000000.0, op-cost: 1000000.0, total-cost: 1000000.0]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_add/object_add.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_add/object_add.3.adm
index 800859b..94bb2b5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_add/object_add.3.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_add/object_add.3.adm
@@ -1 +1 @@
-{ "t1": [ true, true, true, true, true, true, true ], "t2": { "a": 1, "field": "value" }, "t3": { "a": 1, "field": null }, "t4": { "a": 1, "field": { "x": [ "y", "z" ] } }, "t5": { "a": 1 }, "t6": { "a": 1 }, "t7": [ { "screen-name": "ChangEwing_573", "lang": "en", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136, "field": "value" } ], "t8": [ { "tweetid": "1", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "send-time": datetime("2008-04-26T10:10:00.000"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)", "field": { "a": 1 } } ] }
+{ "t1": [ true, true, true, true, true, true, true ], "t2": { "a": 1, "field": "value" }, "t3": { "a": 1, "field": null }, "t4": { "a": 1, "field": { "x": [ "y", "z" ] } }, "t5": { "a": 1 }, "t6": { "a": 1 }, "t7": [ { "screen-name": "ChangEwing_573", "lang": "en", "friends_count": 182, "statuses_count": 394, "field": "value", "name": "Chang Ewing", "followers_count": 32136 } ], "t8": [ { "tweetid": "1", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "send-time": datetime("2008-04-26T10:10:00.000"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)", "field": { "a": 1 } } ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_add/object_add.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_add/object_add.7.adm
new file mode 100644
index 0000000..3bbbabb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_add/object_add.7.adm
@@ -0,0 +1 @@
+{ "t1c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "name": "John" }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "name": "John" }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "name": "John" } ], "t1o": [ { "_id": 4, "name": "John", "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "name": "John", "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "name": "John", "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t2c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } } ], "t2o": [ { "_id": 4, "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t3c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } } ], "t3o": [ { "_id": 4, "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] }, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] }, "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] }, "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t4c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ] }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ] }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ] } ], "t4o": [ { "_id": 4, "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ], "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ], "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ], "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t5c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ] }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ] }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ] } ], "t5o": [ { "_id": 4, "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ], "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ], "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ], "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_concat/object_concat.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_concat/object_concat.5.adm
new file mode 100644
index 0000000..e20e752
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_concat/object_concat.5.adm
@@ -0,0 +1 @@
+{ "res": { "id": 2, "f": 4 } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_concat/object_concat.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_concat/object_concat.6.adm
new file mode 100644
index 0000000..80023c6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_concat/object_concat.6.adm
@@ -0,0 +1 @@
+{ "id": 1, "b1": 3, "dup": 5, "a1": 3 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_put/object_put.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_put/object_put.3.adm
index 4b97741..2947405 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_put/object_put.3.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_put/object_put.3.adm
@@ -1 +1 @@
-{ "t1": [ true, true, true, true, true, true, true ], "t2": { "a": 1, "field": "value" }, "t3": { "a": 1, "field": null }, "t4": { "a": 1, "field": { "x": [ "y", "z" ] } }, "t5": { "a": "replaced" }, "t6": { "a": 1 }, "t7": [ { "screen-name": "ChangEwing_573", "lang": "en", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136, "field": "value" } ], "t8": [ { "tweetid": "1", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "send-time": datetime("2008-04-26T10:10:00.000"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)", "field": { "a": 1 } } ] }
+{ "t1": [ true, true, true, true, true, true, true ], "t2": { "a": 1, "field": "value" }, "t3": { "a": 1, "field": null }, "t4": { "a": 1, "field": { "x": [ "y", "z" ] } }, "t5": { "a": "replaced" }, "t6": { "a": 1 }, "t7": [ { "screen-name": "ChangEwing_573", "lang": "en", "friends_count": 182, "statuses_count": 394, "field": "value", "name": "Chang Ewing", "followers_count": 32136 } ], "t8": [ { "tweetid": "1", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "send-time": datetime("2008-04-26T10:10:00.000"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)", "field": { "a": 1 } } ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_put/object_put.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_put/object_put.7.adm
new file mode 100644
index 0000000..32b297d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/object_put/object_put.7.adm
@@ -0,0 +1 @@
+{ "t1c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "name": "John" }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "name": "John" }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "name": "John" } ], "t1o": [ { "_id": 4, "name": "John", "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "name": "John", "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "name": "John", "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t2c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } } ], "t2o": [ { "_id": 4, "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriend": { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t3c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } } ], "t3o": [ { "_id": 4, "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] }, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] }, "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriend": { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] }, "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t4c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ] }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ] }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ] } ], "t4o": [ { "_id": 4, "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ], "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ], "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriends": [ { "_id": 4, "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] } ], "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t5c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ], "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ] }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 1 } ], "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ] }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 1 }, { "_id": 2 } ], "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ] } ], "t5o": [ { "_id": 4, "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ], "favorite_color": "Green", "best_friend": { "_id": 4 }, "friends": [ ] }, { "_id": 5, "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ], "favorite_color": "Blue", "best_friend": { "_id": 4 }, "friends": [ { "_id": 4 } ] }, { "_id": 6, "newFriends": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ ] } ], "favorite_color": "Orange", "best_friend": { "_id": 5 }, "friends": [ { "_id": 4 }, { "_id": 5 } ] } ], "t6c": [ { "_id": 1, "best_friend": { "_id": 1 }, "friends": [ { "_id": 8 }, { "_id": 9 } ] }, { "_id": 2, "best_friend": { "_id": 1 }, "friends": [ { "_id": 8 }, { "_id": 9 } ] }, { "_id": 3, "best_friend": { "_id": 2 }, "friends": [ { "_id": 8 }, { "_id": 9 } ] } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan/full-scan.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan/full-scan.3.regexjson
index c6f99d1b..0c4fae1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan/full-scan.3.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan/full-scan.3.regexjson
@@ -1,5 +1,9 @@
{
"job-id": "R{[A-Z0-9.:]+}",
+ "create-time": "R{[0-9.]+}",
+ "start-time": "R{[0-9.]+}",
+ "queued-time": "R{.+}",
+ "end-time": "R{[0-9.]+}",
"counters": [],
"joblets": [
{
@@ -29,19 +33,29 @@
],
"counters": [
{
- "name": "Empty Tuple Source",
+ "name": "R{.+}",
"time": "R{[0-9.]+}",
- "disk-io": "R{[0-9.]+}"
- },
- {
- "name": "Index Search",
- "time": "R{[0-9.]+}",
- "disk-io": "R{[0-9.]+}"
+ "runtime-id": "R{.+}"
},
{
"name": "R{.+}",
"time": "R{[0-9.]+}",
- "disk-io": "R{[0-9.]+}"
+ "runtime-id": "R{.+}",
+ "pages-read": "R{[0-9.]+}",
+ "pages-read-cold": "R{[0-9.]+}",
+ "cardinality-out": "R{[0-9.]+}",
+ "avg-tuple-size": "R{[0-9.]+}",
+ "min-tuple-size": "R{[0-9.]+}",
+ "max-tuple-size": "R{[0-9.]+}"
+ },
+ {
+ "name": "R{.+}",
+ "time": "R{[0-9.]+}",
+ "runtime-id": "R{.+}",
+ "cardinality-out": "R{[0-9.]+}",
+ "avg-tuple-size": "R{[0-9.]+}",
+ "min-tuple-size": "R{[0-9.]+}",
+ "max-tuple-size": "R{[0-9.]+}"
}
]
},
@@ -52,18 +66,21 @@
"partition-send-profile": [],
"counters": [
{
- "name": "R{.+}",
- "time": "R{[0-9.]+}",
- "disk-io": "R{[0-9.]+}"
+ "name": "R{.+}",
+ "time": "R{[0-9.]+}",
+ "runtime-id": "R{.+}",
+ "cardinality-out": "R{[0-9.]+}",
+ "avg-tuple-size": "R{[0-9.]+}",
+ "min-tuple-size": "R{[0-9.]+}",
+ "max-tuple-size": "R{[0-9.]+}"
},
{
- "name": "Result Writer",
- "time": "R{[0-9.]+}",
- "disk-io": "R{[0-9.]+}"
+ "name": "R{.+}",
+ "time": "R{[0-9.]+}",
+ "runtime-id": "R{.+}"
}
]
}
]
- }
- ]
+ }]
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.plan
index 6faed2f..c69e8a0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.plan
@@ -5,28 +5,28 @@
project ([$$l])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$16(ASC), $$17(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$17(ASC), $$18(ASC) ] |PARTITIONED|
select (eq($$l.getField(10), "1994-01-20"))
-- STREAM_SELECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- unnest-map [$$16, $$17, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$24, $$25, 2, $$24, $$25, true, true, true)
+ unnest-map [$$17, $$18, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$25, $$26, 2, $$25, $$26, true, true, true)
-- BTREE_SEARCH |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- order (ASC, $$24) (ASC, $$25)
- -- STABLE_SORT [$$24(ASC), $$25(ASC)] |PARTITIONED|
+ order (ASC, $$25) (ASC, $$26)
+ -- STABLE_SORT [$$25(ASC), $$26(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$24, $$25])
+ project ([$$25, $$26])
-- STREAM_PROJECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- unnest-map [$$23, $$24, $$25] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$21, 1, $$22, true, true, true)
+ unnest-map [$$24, $$25, $$26] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$22, 1, $$23, true, true, true)
-- BTREE_SEARCH |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$21, $$22] <- ["1994-01-20", "1994-01-20"]
+ assign [$$22, $$23] <- ["1994-01-20", "1994-01-20"]
-- ASSIGN |PARTITIONED|
empty-tuple-source
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.plan
index 6faed2f..c69e8a0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.plan
@@ -5,28 +5,28 @@
project ([$$l])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$16(ASC), $$17(ASC) ] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$17(ASC), $$18(ASC) ] |PARTITIONED|
select (eq($$l.getField(10), "1994-01-20"))
-- STREAM_SELECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- unnest-map [$$16, $$17, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$24, $$25, 2, $$24, $$25, true, true, true)
+ unnest-map [$$17, $$18, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$25, $$26, 2, $$25, $$26, true, true, true)
-- BTREE_SEARCH |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- order (ASC, $$24) (ASC, $$25)
- -- STABLE_SORT [$$24(ASC), $$25(ASC)] |PARTITIONED|
+ order (ASC, $$25) (ASC, $$26)
+ -- STABLE_SORT [$$25(ASC), $$26(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$24, $$25])
+ project ([$$25, $$26])
-- STREAM_PROJECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- unnest-map [$$23, $$24, $$25] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$21, 1, $$22, true, true, true)
+ unnest-map [$$24, $$25, $$26] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$22, 1, $$23, true, true, true)
-- BTREE_SEARCH |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$21, $$22] <- ["1994-01-20", "1994-01-20"]
+ assign [$$22, $$23] <- ["1994-01-20", "1994-01-20"]
-- ASSIGN |PARTITIONED|
empty-tuple-source
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.10.adm
new file mode 100644
index 0000000..254828e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.10.adm
@@ -0,0 +1,4 @@
+{ "U1": { "user_id": 1, "best_friend": 2, "phones": [ { "kind": "MOBILE", "number": "222-222-2222" } ] }, "U2": { "user_id": 2, "best_friend": 1 } }
+{ "U1": { "user_id": 2, "best_friend": 1 }, "U2": { "user_id": 1, "best_friend": 2, "phones": [ { "kind": "MOBILE", "number": "222-222-2222" } ] } }
+{ "U1": { "user_id": 3, "best_friend": 1, "favorite_color": "Green" }, "U2": { "user_id": 1, "best_friend": 2, "phones": [ { "kind": "MOBILE", "number": "222-222-2222" } ] } }
+{ "U1": { "user_id": 4, "best_friend": null } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.11.adm
new file mode 100644
index 0000000..a177d29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.11.adm
@@ -0,0 +1 @@
+{ "best_friend": 1 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.12.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.12.adm
new file mode 100644
index 0000000..eab8cee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.12.adm
@@ -0,0 +1,4 @@
+{ "user_id": 1, "best_friend": 2 }
+{ "user_id": 2, "best_friend": 1 }
+{ "user_id": 3, "best_friend": 1 }
+{ "user_id": 4, "best_friend": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.13.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.13.adm
new file mode 100644
index 0000000..62782bc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.13.adm
@@ -0,0 +1,3 @@
+{ "A": { "b": 2, "EXCLUDE": 3 } }
+{ "EXCLUDE": 3 }
+{ "EXCLUDE": { "a": 1, "b": 2, "EXCLUDE": 3 } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.14.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.14.adm
new file mode 100644
index 0000000..d44fb9c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.14.adm
@@ -0,0 +1,2 @@
+{ "user_id": 1, "best_friend": 2, "bestFriends": [ { "user_id": 2, "best_friend": 1, "address": { "zip_code": "99929", "street": "2341 Apple Street" } } ] }
+{ "user_id": 2, "best_friend": 1, "bestFriends": [ { "user_id": 1, "best_friend": 2, "phones": [ { "kind": "MOBILE", "number": "222-222-2222" } ] } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.3.adm
new file mode 100644
index 0000000..f72a7e5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.3.adm
@@ -0,0 +1,4 @@
+{ "user_id": 1, "best_friend": 2 }
+{ "user_id": 2, "best_friend": 1 }
+{ "user_id": 3, "best_friend": 1, "favorite_color": "Green" }
+{ "user_id": 4, "best_friend": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.4.adm
new file mode 100644
index 0000000..ae07f96
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.4.adm
@@ -0,0 +1,4 @@
+{ "user_id": 1, "best_friend": 2, "phones": [ { "kind": "MOBILE", "number": "222-222-2222" } ] }
+{ "user_id": 2, "best_friend": 1 }
+{ "user_id": 3, "best_friend": 1, "favorite_color": "Green" }
+{ "user_id": 4, "best_friend": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.5.adm
new file mode 100644
index 0000000..16d3516
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.5.adm
@@ -0,0 +1,2 @@
+[ { "user_id": 2, "best_friend": 1 } ]
+[ { "user_id": 3, "best_friend": 1, "favorite_color": "Green" } ]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.6.adm
new file mode 100644
index 0000000..bf58f6a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.6.adm
@@ -0,0 +1,4 @@
+{ "user_id": 1 }
+{ "user_id": 2 }
+{ "user_id": 3 }
+{ "user_id": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.7.adm
new file mode 100644
index 0000000..71d7a13
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.7.adm
@@ -0,0 +1,3 @@
+{ "best_friend": 1 }
+{ "best_friend": 2 }
+{ "best_friend": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.8.adm
new file mode 100644
index 0000000..3c5ba18
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.8.adm
@@ -0,0 +1 @@
+{ "user_id": 1, "best_friend": 2 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.9.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.9.adm
new file mode 100644
index 0000000..c8cbba5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/complex-exclude/complex-exclude.9.adm
@@ -0,0 +1,4 @@
+{ "U": { "user_id": 1, "best_friend": 2 }, "miscInfo": { "24as": "23412" } }
+{ "U": { "user_id": 2, "best_friend": 1 }, "miscInfo": { "24as": "23412" } }
+{ "U": { "user_id": 3, "best_friend": 1, "favorite_color": "Green" }, "miscInfo": { "24as": "23412" } }
+{ "U": { "user_id": 4, "best_friend": null }, "miscInfo": { "24as": "23412" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.3.adm
new file mode 100644
index 0000000..c27da48
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.3.adm
@@ -0,0 +1,12 @@
+{ "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)" }
+{ "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("29.15,76.53"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " hate verizon its voice-clarity is OMG:(" }
+{ "user": { "screen-name": "NilaMilliron_tw", "friends_count": 445, "statuses_count": 164, "name": "Nila Milliron", "followers_count": 22649 }, "sender-location": point("37.59,68.42"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " can't stand iphone its platform is terrible" }
+{ "user": { "screen-name": "OliJackson_512", "friends_count": 445, "statuses_count": 164, "name": "Oli Jackson", "followers_count": 22649 }, "sender-location": point("24.82,94.63"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " like samsung the voice-command is amazing:)" }
+{ "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("32.84,67.14"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon its shortcut-menu is awesome:)" }
+{ "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("29.72,75.8"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " like motorola the speed is good:)" }
+{ "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("39.28,70.48"), "referred-topics": {{ "sprint", "voice-command" }}, "message-text": " like sprint the voice-command is mind-blowing:)" }
+{ "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("40.09,92.69"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " can't stand motorola its speed is terrible:(" }
+{ "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("47.51,83.99"), "referred-topics": {{ "iphone", "voice-clarity" }}, "message-text": " like iphone the voice-clarity is good:)" }
+{ "user": { "screen-name": "ChangEwing_573", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136 }, "sender-location": point("36.21,72.6"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is good" }
+{ "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("46.05,93.34"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is awesome:)" }
+{ "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("36.86,74.62"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.4.adm
new file mode 100644
index 0000000..0b3ee31
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.4.adm
@@ -0,0 +1,11 @@
+{ "TM": { "tweetid": "1", "user": { "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "send-time": datetime("2008-04-26T10:10:00.000"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)" } }
+{ "TM": { "tweetid": "10", "user": { "lang": "en", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("29.15,76.53"), "send-time": datetime("2008-01-26T10:10:00.000"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " hate verizon its voice-clarity is OMG:(" } }
+{ "TM": { "tweetid": "11", "user": { "lang": "en", "friends_count": 445, "statuses_count": 164, "name": "Nila Milliron", "followers_count": 22649 }, "sender-location": point("37.59,68.42"), "send-time": datetime("2008-03-09T10:10:00.000"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " can't stand iphone its platform is terrible" } }
+{ "TM": { "tweetid": "2", "user": { "lang": "en", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("32.84,67.14"), "send-time": datetime("2010-05-13T10:10:00.000"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon its shortcut-menu is awesome:)" } }
+{ "TM": { "tweetid": "3", "user": { "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("29.72,75.8"), "send-time": datetime("2006-11-04T10:10:00.000"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " like motorola the speed is good:)" } }
+{ "TM": { "tweetid": "4", "user": { "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("39.28,70.48"), "send-time": datetime("2011-12-26T10:10:00.000"), "referred-topics": {{ "sprint", "voice-command" }}, "message-text": " like sprint the voice-command is mind-blowing:)" } }
+{ "TM": { "tweetid": "5", "user": { "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("40.09,92.69"), "send-time": datetime("2006-08-04T10:10:00.000"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " can't stand motorola its speed is terrible:(" } }
+{ "TM": { "tweetid": "6", "user": { "lang": "en", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("47.51,83.99"), "send-time": datetime("2010-05-07T10:10:00.000"), "referred-topics": {{ "iphone", "voice-clarity" }}, "message-text": " like iphone the voice-clarity is good:)" } }
+{ "TM": { "tweetid": "7", "user": { "lang": "en", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136 }, "sender-location": point("36.21,72.6"), "send-time": datetime("2011-08-25T10:10:00.000"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is good" } }
+{ "TM": { "tweetid": "8", "user": { "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("46.05,93.34"), "send-time": datetime("2005-10-14T10:10:00.000"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is awesome:)" } }
+{ "TM": { "tweetid": "9", "user": { "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("36.86,74.62"), "send-time": datetime("2012-07-21T10:10:00.000"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.5.adm
new file mode 100644
index 0000000..0621911
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.5.adm
@@ -0,0 +1,24 @@
+{ "tweetid": "1", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)" }
+{ "tweetid": "1", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)", "send-time": datetime("2008-04-26T10:10:00.000") }
+{ "tweetid": "10", "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("29.15,76.53"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " hate verizon its voice-clarity is OMG:(" }
+{ "tweetid": "10", "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("29.15,76.53"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " hate verizon its voice-clarity is OMG:(", "send-time": datetime("2008-01-26T10:10:00.000") }
+{ "tweetid": "11", "user": { "screen-name": "NilaMilliron_tw", "friends_count": 445, "statuses_count": 164, "name": "Nila Milliron", "followers_count": 22649 }, "sender-location": point("37.59,68.42"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " can't stand iphone its platform is terrible" }
+{ "tweetid": "11", "user": { "screen-name": "NilaMilliron_tw", "friends_count": 445, "statuses_count": 164, "name": "Nila Milliron", "followers_count": 22649 }, "sender-location": point("37.59,68.42"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " can't stand iphone its platform is terrible", "send-time": datetime("2008-03-09T10:10:00.000") }
+{ "tweetid": "12", "user": { "screen-name": "OliJackson_512", "friends_count": 445, "statuses_count": 164, "name": "Oli Jackson", "followers_count": 22649 }, "sender-location": point("24.82,94.63"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " like samsung the voice-command is amazing:)" }
+{ "tweetid": "12", "user": { "screen-name": "OliJackson_512", "friends_count": 445, "statuses_count": 164, "name": "Oli Jackson", "followers_count": 22649 }, "sender-location": point("24.82,94.63"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " like samsung the voice-command is amazing:)", "send-time": datetime("2010-02-13T10:10:00.000") }
+{ "tweetid": "2", "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("32.84,67.14"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon its shortcut-menu is awesome:)" }
+{ "tweetid": "2", "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("32.84,67.14"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon its shortcut-menu is awesome:)", "send-time": datetime("2010-05-13T10:10:00.000") }
+{ "tweetid": "3", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("29.72,75.8"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " like motorola the speed is good:)" }
+{ "tweetid": "3", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("29.72,75.8"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " like motorola the speed is good:)", "send-time": datetime("2006-11-04T10:10:00.000") }
+{ "tweetid": "4", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("39.28,70.48"), "referred-topics": {{ "sprint", "voice-command" }}, "message-text": " like sprint the voice-command is mind-blowing:)" }
+{ "tweetid": "4", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("39.28,70.48"), "referred-topics": {{ "sprint", "voice-command" }}, "message-text": " like sprint the voice-command is mind-blowing:)", "send-time": datetime("2011-12-26T10:10:00.000") }
+{ "tweetid": "5", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("40.09,92.69"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " can't stand motorola its speed is terrible:(" }
+{ "tweetid": "5", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("40.09,92.69"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " can't stand motorola its speed is terrible:(", "send-time": datetime("2006-08-04T10:10:00.000") }
+{ "tweetid": "6", "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("47.51,83.99"), "referred-topics": {{ "iphone", "voice-clarity" }}, "message-text": " like iphone the voice-clarity is good:)" }
+{ "tweetid": "6", "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("47.51,83.99"), "referred-topics": {{ "iphone", "voice-clarity" }}, "message-text": " like iphone the voice-clarity is good:)", "send-time": datetime("2010-05-07T10:10:00.000") }
+{ "tweetid": "7", "user": { "screen-name": "ChangEwing_573", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136 }, "sender-location": point("36.21,72.6"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is good" }
+{ "tweetid": "7", "user": { "screen-name": "ChangEwing_573", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136 }, "sender-location": point("36.21,72.6"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is good", "send-time": datetime("2011-08-25T10:10:00.000") }
+{ "tweetid": "8", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("46.05,93.34"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is awesome:)" }
+{ "tweetid": "8", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("46.05,93.34"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is awesome:)", "send-time": datetime("2005-10-14T10:10:00.000") }
+{ "tweetid": "9", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("36.86,74.62"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome" }
+{ "tweetid": "9", "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("36.86,74.62"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome", "send-time": datetime("2012-07-21T10:10:00.000") }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.6.adm
new file mode 100644
index 0000000..81592e9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-exclude/tiny-social/tiny-social.6.adm
@@ -0,0 +1,12 @@
+{ "TM": { "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)" } }
+{ "TM": { "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("29.15,76.53"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " hate verizon its voice-clarity is OMG:(" } }
+{ "TM": { "user": { "screen-name": "NilaMilliron_tw", "friends_count": 445, "statuses_count": 164, "name": "Nila Milliron", "followers_count": 22649 }, "sender-location": point("37.59,68.42"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " can't stand iphone its platform is terrible" } }
+{ "TM": { "user": { "screen-name": "OliJackson_512", "friends_count": 445, "statuses_count": 164, "name": "Oli Jackson", "followers_count": 22649 }, "sender-location": point("24.82,94.63"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " like samsung the voice-command is amazing:)" } }
+{ "TM": { "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("32.84,67.14"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon its shortcut-menu is awesome:)" } }
+{ "TM": { "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("29.72,75.8"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " like motorola the speed is good:)" } }
+{ "TM": { "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("39.28,70.48"), "referred-topics": {{ "sprint", "voice-command" }}, "message-text": " like sprint the voice-command is mind-blowing:)" } }
+{ "TM": { "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("40.09,92.69"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " can't stand motorola its speed is terrible:(" } }
+{ "TM": { "user": { "screen-name": "ColineGeyer@63", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("47.51,83.99"), "referred-topics": {{ "iphone", "voice-clarity" }}, "message-text": " like iphone the voice-clarity is good:)" } }
+{ "TM": { "user": { "screen-name": "ChangEwing_573", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136 }, "sender-location": point("36.21,72.6"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is good" } }
+{ "TM": { "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("46.05,93.34"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is awesome:)" } }
+{ "TM": { "user": { "screen-name": "NathanGiesen@211", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("36.86,74.62"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.3.adm
new file mode 100644
index 0000000..cefb946
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.3.adm
@@ -0,0 +1,40 @@
+{ "ds": "message1", "id": 1, "no_in_response_to": { "message-id": 1, "author-id": 3, "sender-location": point("47.16,77.75"), "message": " love sprint its shortcut-menu is awesome:)" } }
+{ "ds": "message2", "id": 1, "no_author_id": { "message-id": 1, "in-response-to": 2, "sender-location": point("47.16,77.75"), "message": " love sprint its shortcut-menu is awesome:)" } }
+{ "ds": "user", "id": 1, "user": { "id": 1, "alias": "Margarita", "name": "MargaritaStoddard", "user-since": datetime("2012-08-20T10:10:00.000"), "friend-ids": {{ 2, 3, 6, 10 }}, "employment": [ { "organization-name": "Codetechno", "start-date": date("2006-08-06") } ] } }
+{ "ds": "message1", "id": 2, "no_in_response_to": { "message-id": 2, "author-id": 1, "sender-location": point("41.66,80.87"), "message": " dislike iphone its touch-screen is horrible" } }
+{ "ds": "message2", "id": 2, "no_author_id": { "message-id": 2, "in-response-to": 4, "sender-location": point("41.66,80.87"), "message": " dislike iphone its touch-screen is horrible" } }
+{ "ds": "user", "id": 2, "user": { "id": 2, "alias": "Isbel", "name": "IsbelDull", "user-since": datetime("2011-01-22T10:10:00.000"), "friend-ids": {{ 1, 4 }}, "employment": [ { "organization-name": "Hexviafind", "start-date": date("2010-04-27") } ] } }
+{ "ds": "message1", "id": 3, "no_in_response_to": { "message-id": 3, "author-id": 2, "sender-location": point("48.09,81.01"), "message": " like samsung the plan is amazing" } }
+{ "ds": "message2", "id": 3, "no_author_id": { "message-id": 3, "in-response-to": 4, "sender-location": point("48.09,81.01"), "message": " like samsung the plan is amazing" } }
+{ "ds": "user", "id": 3, "user": { "id": 3, "alias": "Emory", "name": "EmoryUnk", "user-since": datetime("2012-07-10T10:10:00.000"), "friend-ids": {{ 1, 5, 8, 9 }}, "employment": [ { "organization-name": "geomedia", "start-date": date("2010-06-17"), "end-date": date("2010-01-26") } ] } }
+{ "ds": "message1", "id": 4, "no_in_response_to": { "message-id": 4, "author-id": 1, "sender-location": point("37.73,97.04"), "message": " can't stand at&t the network is horrible:(" } }
+{ "ds": "message2", "id": 4, "no_author_id": { "message-id": 4, "in-response-to": 2, "sender-location": point("37.73,97.04"), "message": " can't stand at&t the network is horrible:(" } }
+{ "ds": "user", "id": 4, "user": { "id": 4, "alias": "Nicholas", "name": "NicholasStroh", "user-since": datetime("2010-12-27T10:10:00.000"), "friend-ids": {{ 2 }}, "employment": [ { "organization-name": "Zamcorporation", "start-date": date("2010-06-08") } ] } }
+{ "ds": "message1", "id": 5, "no_in_response_to": { "message-id": 5, "author-id": 6, "sender-location": point("34.7,90.76"), "message": " love sprint the customization is mind-blowing" } }
+{ "ds": "message2", "id": 5, "no_author_id": { "message-id": 5, "in-response-to": 2, "sender-location": point("34.7,90.76"), "message": " love sprint the customization is mind-blowing" } }
+{ "ds": "user", "id": 5, "user": { "id": 5, "alias": "Von", "name": "VonKemble", "user-since": datetime("2010-01-05T10:10:00.000"), "friend-ids": {{ 3, 6, 10 }}, "employment": [ { "organization-name": "Kongreen", "start-date": date("2010-11-27") } ] } }
+{ "ds": "message1", "id": 6, "no_in_response_to": { "message-id": 6, "author-id": 2, "sender-location": point("31.5,75.56"), "message": " like t-mobile its platform is mind-blowing" } }
+{ "ds": "message2", "id": 6, "no_author_id": { "message-id": 6, "in-response-to": 1, "sender-location": point("31.5,75.56"), "message": " like t-mobile its platform is mind-blowing" } }
+{ "ds": "user", "id": 6, "user": { "id": 6, "alias": "Willis", "name": "WillisWynne", "user-since": datetime("2005-01-17T10:10:00.000"), "friend-ids": {{ 1, 3, 7 }}, "employment": [ { "organization-name": "jaydax", "start-date": date("2009-05-15") } ] } }
+{ "ds": "message1", "id": 7, "no_in_response_to": { "message-id": 7, "author-id": 5, "sender-location": point("32.91,85.05"), "message": " dislike sprint the speed is horrible" } }
+{ "ds": "message2", "id": 7, "no_author_id": { "message-id": 7, "in-response-to": 15, "sender-location": point("32.91,85.05"), "message": " dislike sprint the speed is horrible" } }
+{ "ds": "user", "id": 7, "user": { "id": 7, "alias": "Suzanna", "name": "SuzannaTillson", "user-since": datetime("2012-08-07T10:10:00.000"), "friend-ids": {{ 6 }}, "employment": [ { "organization-name": "Labzatron", "start-date": date("2011-04-19") } ] } }
+{ "ds": "message1", "id": 8, "no_in_response_to": { "message-id": 8, "author-id": 1, "sender-location": point("40.33,80.87"), "message": " like verizon the 3G is awesome:)" } }
+{ "ds": "message2", "id": 8, "no_author_id": { "message-id": 8, "in-response-to": 11, "sender-location": point("40.33,80.87"), "message": " like verizon the 3G is awesome:)" } }
+{ "ds": "user", "id": 8, "user": { "id": 8, "alias": "Nila", "name": "NilaMilliron", "user-since": datetime("2008-01-01T10:10:00.000"), "friend-ids": {{ 3 }}, "employment": [ { "organization-name": "Plexlane", "start-date": date("2010-02-28") } ] } }
+{ "ds": "message1", "id": 9, "no_in_response_to": { "message-id": 9, "author-id": 3, "sender-location": point("34.45,96.48"), "message": " love verizon its wireless is good" } }
+{ "ds": "message2", "id": 9, "no_author_id": { "message-id": 9, "in-response-to": 12, "sender-location": point("34.45,96.48"), "message": " love verizon its wireless is good" } }
+{ "ds": "user", "id": 9, "user": { "id": 9, "alias": "Woodrow", "name": "WoodrowNehling", "user-since": datetime("2005-09-20T10:10:00.000"), "friend-ids": {{ 3, 10 }}, "employment": [ { "organization-name": "Zuncan", "start-date": date("2003-04-22"), "end-date": date("2009-12-13") } ] } }
+{ "ds": "message1", "id": 10, "no_in_response_to": { "message-id": 10, "author-id": 1, "sender-location": point("42.5,70.01"), "message": " can't stand motorola the touch-screen is terrible" } }
+{ "ds": "message2", "id": 10, "no_author_id": { "message-id": 10, "in-response-to": 12, "sender-location": point("42.5,70.01"), "message": " can't stand motorola the touch-screen is terrible" } }
+{ "ds": "user", "id": 10, "user": { "id": 10, "alias": "Bram", "name": "BramHatch", "user-since": datetime("2010-10-16T10:10:00.000"), "friend-ids": {{ 1, 5, 9 }}, "employment": [ { "organization-name": "physcane", "start-date": date("2007-06-05"), "end-date": date("2011-11-05") } ] } }
+{ "ds": "message1", "id": 11, "no_in_response_to": { "message-id": 11, "author-id": 1, "sender-location": point("38.97,77.49"), "message": " can't stand at&t its plan is terrible" } }
+{ "ds": "message2", "id": 11, "no_author_id": { "message-id": 11, "in-response-to": 1, "sender-location": point("38.97,77.49"), "message": " can't stand at&t its plan is terrible" } }
+{ "ds": "message1", "id": 12, "no_in_response_to": { "message-id": 12, "author-id": 10, "sender-location": point("42.26,77.76"), "message": " can't stand t-mobile its voicemail-service is OMG:(" } }
+{ "ds": "message2", "id": 12, "no_author_id": { "message-id": 12, "in-response-to": 6, "sender-location": point("42.26,77.76"), "message": " can't stand t-mobile its voicemail-service is OMG:(" } }
+{ "ds": "message1", "id": 13, "no_in_response_to": { "message-id": 13, "author-id": 10, "sender-location": point("42.77,78.92"), "message": " dislike iphone the voice-command is bad:(" } }
+{ "ds": "message2", "id": 13, "no_author_id": { "message-id": 13, "in-response-to": 4, "sender-location": point("42.77,78.92"), "message": " dislike iphone the voice-command is bad:(" } }
+{ "ds": "message1", "id": 14, "no_in_response_to": { "message-id": 14, "author-id": 9, "sender-location": point("41.33,85.28"), "message": " love at&t its 3G is good:)" } }
+{ "ds": "message2", "id": 14, "no_author_id": { "message-id": 14, "in-response-to": 12, "sender-location": point("41.33,85.28"), "message": " love at&t its 3G is good:)" } }
+{ "ds": "message1", "id": 15, "no_in_response_to": { "message-id": 15, "author-id": 7, "sender-location": point("44.47,67.11"), "message": " like iphone the voicemail-service is awesome" } }
+{ "ds": "message2", "id": 15, "no_author_id": { "message-id": 15, "in-response-to": 11, "sender-location": point("44.47,67.11"), "message": " like iphone the voicemail-service is awesome" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.4.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.4.plan
new file mode 100644
index 0000000..d0aa765
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.4.plan
@@ -0,0 +1,72 @@
+distribute result [$#1] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$#1]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$102(ASC) ] |PARTITIONED|
+ order (ASC, $$102) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$102(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ union ($$64, $$140, $#1) ($$103, $$70, $$102) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNION_ALL |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ union ($$139, $$141, $$64) ($$68, $$69, $$103) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNION_ALL |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$139, $$68]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$139] <- [cast({"id": $$68, "no_in_response_to": object-remove($$s, "in-response-to")})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ replicate [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- REPLICATE |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$68, $$s] <- TinySocial.FacebookMessages [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$141, $$69]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$141] <- [cast({"id": $$69, "user": $$t})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$69, $$t] <- TinySocial.FacebookUsers [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$140, $$70]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$140] <- [cast({"id": $$70, "no_author_id": object-remove($$s, "author-id")})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ project ([$$70, $$s]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$70, $$s] <- [$$68, $$s] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ replicate [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- REPLICATE |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$68, $$s] <- TinySocial.FacebookMessages [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.03.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.03.regexadm
index 519019a..7029ce2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.03.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.03.regexadm
@@ -31,6 +31,7 @@
\s*\Q"metrics": {\E
\s*\Q"elapsedTime": "\E[^"]+\Q",\E
\s*\Q"executionTime": "\E[^"]+\Q",\E
+\s*\Q"compileTime": "\E[^"]+\Q",\E
\s*\Q"resultCount": \E[0-9]+\Q,\E
\s*\Q"resultSize": \E[0-9]+\Q,\E
\s*\Q"processedObjects": \E[0-9]+\Q,\E
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.04.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.04.regexadm
index 8eb28ea..8478d17 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.04.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.04.regexadm
@@ -24,6 +24,7 @@
\s*\Q"metrics": {\E
\s*\Q"elapsedTime": "\E[^"]+\Q",\E
\s*\Q"executionTime": "\E[^"]+\Q",\E
+\s*\Q"compileTime": "\E[^"]+\Q",\E
\s*\Q"resultCount": \E[0-9]+\Q,\E
\s*\Q"resultSize": \E[0-9]+\Q,\E
\s*\Q"processedObjects": \E[0-9]+\Q,\E
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.05.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.05.regexadm
index 5c477e7..5399ed8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.05.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.05.regexadm
@@ -24,6 +24,7 @@
\s*\Q"metrics": {\E
\s*\Q"elapsedTime": "\E[^"]+\Q",\E
\s*\Q"executionTime": "\E[^"]+\Q",\E
+\s*\Q"compileTime": "\E[^"]+\Q",\E
\s*\Q"resultCount": \E[0-9]+\Q,\E
\s*\Q"resultSize": \E[0-9]+\Q,\E
\s*\Q"processedObjects": \E[0-9]+\Q,\E
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.06.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.06.regexadm
index 5eeb3df..aa58fb4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.06.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.06.regexadm
@@ -12,6 +12,7 @@
\s*\Q"metrics": {\E
\s*\Q"elapsedTime": "\E[^"]+\Q",\E
\s*\Q"executionTime": "\E[^"]+\Q",\E
+\s*\Q"compileTime": "\E[^"]+\Q",\E
\s*\Q"resultCount": \E[0-9]+\Q,\E
\s*\Q"resultSize": \E[0-9]+\Q,\E
\s*\Q"processedObjects": \E[0-9]+\Q,\E
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.07.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.07.regexadm
index 5c477e7..5399ed8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.07.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.07.regexadm
@@ -24,6 +24,7 @@
\s*\Q"metrics": {\E
\s*\Q"elapsedTime": "\E[^"]+\Q",\E
\s*\Q"executionTime": "\E[^"]+\Q",\E
+\s*\Q"compileTime": "\E[^"]+\Q",\E
\s*\Q"resultCount": \E[0-9]+\Q,\E
\s*\Q"resultSize": \E[0-9]+\Q,\E
\s*\Q"processedObjects": \E[0-9]+\Q,\E
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.08.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.08.regexadm
index 519019a..7029ce2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.08.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/warnings/warnings-limit/warnings-limit.08.regexadm
@@ -31,6 +31,7 @@
\s*\Q"metrics": {\E
\s*\Q"elapsedTime": "\E[^"]+\Q",\E
\s*\Q"executionTime": "\E[^"]+\Q",\E
+\s*\Q"compileTime": "\E[^"]+\Q",\E
\s*\Q"resultCount": \E[0-9]+\Q,\E
\s*\Q"resultSize": \E[0-9]+\Q,\E
\s*\Q"processedObjects": \E[0-9]+\Q,\E
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
index 686ede2..284c2fd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -52,15 +52,7 @@
<test-case FilePath="external-library" check-warnings="true">
<compilation-unit name="py_function_error">
<output-dir compare="Clean-JSON">py_function_error</output-dir>
- <expected-warn>ASX0201: External UDF returned exception. Returned exception was: Traceback (most recent call last):
- File "entrypoint.py", line 181, in handle_call
- result[0].append(self.next_tuple(*arg, key=self.mid))
- File "entrypoint.py", line 99, in next_tuple
- return self.wrapped_fns[key](*args)
- File "site-packages/roundtrip.py", line 32, in warning
- raise ArithmeticError("oof")
-ArithmeticError: oof
- (in line 28, at column 1)</expected-warn>
+ <expected-warn>ArithmeticError: oof</expected-warn>
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
@@ -76,8 +68,8 @@
<test-case FilePath="external-library" check-warnings="true">
<compilation-unit name="crash">
<output-dir compare="Text">crash</output-dir>
- <expected-warn>ASX0201: External UDF returned exception. Returned exception was: Function externallibtest:crash#0 failed to execute (in line 23, at column 1)</expected-warn>
- <expected-warn>ASX0201: External UDF returned exception. Returned exception was: java.io.IOException: Python process exited with code: 1 (in line 23, at column 1)</expected-warn>
+ <expected-warn>ASX0201: External UDF returned exception.</expected-warn>
+ <expected-warn>ASX0201: External UDF returned exception.</expected-warn>
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index bf2d80f..e0a8fa6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3106,6 +3106,21 @@
<expected-error>Input contains different list types (in line 25, at column 8)</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_binary_search">
+ <output-dir compare="Text">array_binary_search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_move">
+ <output-dir compare="Text">array_move</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_swap">
+ <output-dir compare="Text">array_swap</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="bitwise">
<test-case FilePath="bitwise">
@@ -9965,6 +9980,25 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="select-exclude">
+ <test-case FilePath="select-exclude">
+ <compilation-unit name="tiny-social">
+ <output-dir compare="Text">tiny-social</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-exclude">
+ <compilation-unit name="complex-exclude">
+ <output-dir compare="Text">complex-exclude</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-exclude">
+ <compilation-unit name="exclude-negative">
+ <output-dir compare="Text">exclude-negative</output-dir>
+ <expected-error>ASX1001: Syntax error</expected-error>
+ <expected-error>ASX1001: Syntax error</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="semistructured">
<test-case FilePath="semistructured">
<compilation-unit name="count-nullable">
@@ -15039,6 +15073,11 @@
<output-dir compare="Text">query-ASTERIXDB-1354</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_type_cast">
+ <output-dir compare="Text">union_type_cast</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="upsert">
<test-case FilePath="upsert">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_batch_lookup.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_batch_lookup.xml
new file mode 100644
index 0000000..fd8b886
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_batch_lookup.xml
@@ -0,0 +1,862 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="batched-lookups">
+ <test-case FilePath="dml">
+ <compilation-unit name="compact-dataset-and-its-indexes">
+ <output-dir compare="Text">compact-dataset-and-its-indexes</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-constant-merge-policy">
+ <output-dir compare="Text">using-constant-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-prefix-merge-policy">
+ <output-dir compare="Text">using-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-concurrent-merge-policy">
+ <output-dir compare="Text">using-concurrent-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-correlated-prefix-merge-policy">
+ <output-dir compare="Text">using-correlated-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-correlated-prefix-merge-policy-with-feed">
+ <output-dir compare="Text">using-correlated-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-no-merge-policy">
+ <output-dir compare="Text">using-no-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="delete-from-loaded-dataset-with-index">
+ <output-dir compare="Text">delete-from-loaded-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-index">
+ <output-dir compare="Text">load-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index-open">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-01">
+ <output-dir compare="Text">fulltext-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-02">
+ <output-dir compare="Text">fulltext-index-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-08">
+ <output-dir compare="Text">fulltext-index-08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-09">
+ <output-dir compare="Text">fulltext-index-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="stopwords-full-text-filter-1">
+ <output-dir compare="Text">stopwords-full-text-filter-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-index-nested-loop-join">
+ <output-dir compare="Text">btree-index-nested-loop-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-pidx-to-sidx-idxonly-equi-join_01">
+ <output-dir compare="Text">btree-pidx-to-sidx-idxonly-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-equi-join_02">
+ <output-dir compare="Text">btree-secondary-equi-join_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-equi-join_03">
+ <output-dir compare="Text">btree-secondary-equi-join_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-equi-join_04">
+ <output-dir compare="Text">btree-secondary-equi-join_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-idxonly-to-pidx-equi-join_01">
+ <output-dir compare="Text">btree-sidx-idxonly-to-pidx-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-idxonly-to-sidx-idxonly-equi-join_01">
+ <output-dir compare="Text">btree-sidx-idxonly-to-sidx-idxonly-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-non-idxonly-to-pidx-equi-join_01">
+ <output-dir compare="Text">btree-sidx-non-idxonly-to-pidx-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-non-idxonly-to-sidx-idxonly-equi-join_01">
+ <output-dir compare="Text">btree-sidx-non-idxonly-to-sidx-idxonly-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join" check-warnings="true">
+ <compilation-unit name="hints-indexnl-params">
+ <output-dir compare="Text">hints-indexnl-params</output-dir>
+ <expected-warn>ASX1132: Invalid specification for hint indexnl. ASX1001: Syntax error: In line 1 >>(8, idx_tenk2_1k_2k)<< Encountered <INTEGER_LITERAL> "8" at column 2. (in line 35, at column 21)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="rtree-spatial-intersect-point_05">
+ <output-dir compare="Text">rtree-spatial-intersect-point_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-composite-key-03">
+ <output-dir compare="Text">btree-index-composite-key-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-01">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-02">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-03">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-04">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-idxonly-01">
+ <output-dir compare="Text">btree-sidx-idxonly-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="cust-index-age-nullable">
+ <output-dir compare="Text">cust-index-age-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="intersection-with-between">
+ <output-dir compare="Text">intersection-with-between</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-large-data">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-olist-edit-distance">
+ <output-dir compare="Text">inverted-index-olist-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-olist-jaccard">
+ <output-dir compare="Text">inverted-index-olist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ulist-jaccard">
+ <output-dir compare="Text">inverted-index-ulist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive-open">
+ <output-dir compare="Text">orders-index-custkey-conjunctive-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey-open">
+ <output-dir compare="Text">orders-index-custkey-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="range-search-open">
+ <output-dir compare="Text">range-search-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="rtree-sidx-non-idxonly-02">
+ <output-dir compare="Text">rtree-sidx-non-idxonly-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="dataset-with-meta">
+ <output-dir compare="Text">dataset-with-meta</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection" check-warnings="true">
+ <compilation-unit name="hints-skip-index">
+ <output-dir compare="Text">hints-skip-index</output-dir>
+ <expected-warn>ASX1132: Invalid specification for hint skip-index. ASX1001: Syntax error: In line 1 >>(13, idx_1k)<< Encountered <INTEGER_LITERAL> "13" at column 2. (in line 32, at column 19)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection" check-warnings="true">
+ <compilation-unit name="hints-use-index">
+ <output-dir compare="Text">hints-use-index</output-dir>
+ <expected-warn>ASX1132: Invalid specification for hint use-index. ASX1001: Syntax error: In line 1 >>(18, idx_1k_2k)<< Encountered <INTEGER_LITERAL> "18" at column 2. (in line 33, at column 15)</expected-warn>
+ <expected-warn>ASX1132: Invalid specification for hint use-index. ASX1001: Syntax error: In line 1 >>()<< Encountered ")" at column 2. (in line 33, at column 15)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="verify">
+ <output-dir compare="Text">verify</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="ngram-jaccard">
+ <output-dir compare="Text">ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="ngram-jaccard-inline">
+ <output-dir compare="Text">ngram-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-01">
+ <output-dir compare="Text">btree-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-02">
+ <output-dir compare="Text">btree-index-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-04">
+ <output-dir compare="Text">btree-index-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/correlated-index-selection">
+ <compilation-unit name="btree-index-01">
+ <output-dir compare="Text">btree-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/index-join">
+ <compilation-unit name="btree-equi-join-01">
+ <output-dir compare="Text">btree-equi-join-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="ngram-jaccard">
+ <output-dir compare="Text">ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="ngram-jaccard-inline">
+ <output-dir compare="Text">ngram-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="btree-secondary-non-enforced-equi-join">
+ <output-dir compare="Text">btree-secondary-non-enforced-equi-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="btree-secondary-non-enforced-equi-join-2">
+ <output-dir compare="Text">btree-secondary-non-enforced-equi-join-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="non-enforced-01">
+ <output-dir compare="Text">non-enforced-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="non-enforced-02">
+ <output-dir compare="Text">non-enforced-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="non-enforced-04">
+ <output-dir compare="Text">non-enforced-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="cust-index-age-nullable">
+ <output-dir compare="Text">cust-index-age-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-olist-edit-distance">
+ <output-dir compare="Text">inverted-index-olist-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-olist-jaccard">
+ <output-dir compare="Text">inverted-index-olist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ulist-jaccard">
+ <output-dir compare="Text">inverted-index-ulist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive-open">
+ <output-dir compare="Text">orders-index-custkey-conjunctive-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey-open">
+ <output-dir compare="Text">orders-index-custkey-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="range-search-open">
+ <output-dir compare="Text">range-search-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="rtree-secondary-index-optional">
+ <output-dir compare="Text">rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="compact-dataset-and-its-indexes">
+ <output-dir compare="Text">compact-dataset-and-its-indexes</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="delete-from-loaded-dataset-with-index">
+ <output-dir compare="Text">delete-from-loaded-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="insert-into-loaded-dataset-with-index_01">
+ <output-dir compare="Text">insert-into-loaded-dataset-with-index_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="insert-into-loaded-dataset-with-index_02">
+ <output-dir compare="Text">insert-into-loaded-dataset-with-index_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-ngram-index">
+ <output-dir compare="Text">load-with-ngram-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-rtree-index">
+ <output-dir compare="Text">load-with-rtree-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-word-index">
+ <output-dir compare="Text">load-with-word-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-btree-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="using-feed-new-index">
+ <output-dir compare="Text">using-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="with-open-index">
+ <output-dir compare="Text">with-open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="cell-aggregation-with-filtering">
+ <output-dir compare="Text">cell-aggregation-with-filtering</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="outer_join_01">
+ <output-dir compare="Text">outer_join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="index_01">
+ <output-dir compare="Text">index_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q45">
+ <output-dir compare="Text">q45</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q10_returned_item">
+ <output-dir compare="Text">q10_returned_item</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q10_returned_item_int64">
+ <output-dir compare="Text">q10_returned_item_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q12_shipping">
+ <output-dir compare="Text">q12_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q14_promotion_effect">
+ <output-dir compare="Text">q14_promotion_effect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q15_top_supplier">
+ <output-dir compare="Text">q15_top_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q01_pricing_summary_report_nt">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q03_shipping_priority_nt">
+ <output-dir compare="Text">q03_shipping_priority_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q04_order_priority">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q04_order_priority_with_nodegroup">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q05_local_supplier_volume">
+ <output-dir compare="Text">q05_local_supplier_volume</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q06_forecast_revenue_change">
+ <output-dir compare="Text">q06_forecast_revenue_change</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q07_volume_shipping">
+ <output-dir compare="Text">q07_volume_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q08_national_market_share">
+ <output-dir compare="Text">q08_national_market_share</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue810">
+ <output-dir compare="Text">query-issue810</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue810-2">
+ <output-dir compare="Text">query-issue810-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue810-3">
+ <output-dir compare="Text">query-issue810-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue827-2">
+ <output-dir compare="Text">query-issue827-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-join-btree-sidx3-idxonly">
+ <output-dir compare="Text">probe-pidx-join-btree-sidx3-idxonly</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-idxonly-join-btree-pidx1">
+ <output-dir compare="Text">probe-sidx-btree-idxonly-join-btree-pidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-idxonly-join-btree-sidx1-idxonly">
+ <output-dir compare="Text">probe-sidx-btree-idxonly-join-btree-sidx1-idxonly</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-non-idxonly-join-btree-pidx1">
+ <output-dir compare="Text">probe-sidx-btree-non-idxonly-join-btree-pidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-non-idxonly-join-btree-sidx1-idxonly">
+ <output-dir compare="Text">probe-sidx-btree-non-idxonly-join-btree-sidx1-idxonly</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tinysocial">
+ <compilation-unit name="tinysocial-suite">
+ <output-dir compare="Text">tinysocial-suite</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tinysocial">
+ <compilation-unit name="tinysocial-suite-open">
+ <output-dir compare="Text">tinysocial-suite-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-btree">
+ <output-dir compare="Text">load-with-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-btree-index-only">
+ <output-dir compare="Text">load-with-secondary-btree-index-only</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-inverted-word">
+ <output-dir compare="Text">load-with-secondary-inverted-word</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-rtree">
+ <output-dir compare="Text">load-with-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-btree">
+ <output-dir compare="Text">insert-with-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_opt_1">
+ <output-dir compare="Text">union_opt_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-btree">
+ <output-dir compare="Text">primary-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-primary-lookup">
+ <output-dir compare="Text">push-limit-to-primary-lookup</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-primary-lookup-select">
+ <output-dir compare="Text">push-limit-to-primary-lookup-select</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_profiled.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_profiled.xml
new file mode 100644
index 0000000..c22d6d0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_profiled.xml
@@ -0,0 +1,15880 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<!DOCTYPE test-suite [
+ <!ENTITY ObjectsQueries SYSTEM "queries_sqlpp/objects/ObjectsQueries.xml">
+ <!ENTITY AsyncDeferredQueries SYSTEM "queries_sqlpp/async-deferred/AsyncDeferredQueries.xml">
+ <!ENTITY GeoQueries SYSTEM "queries_sqlpp/geojson/GeoJSONQueries.xml">
+ <!ENTITY TemporalQueries SYSTEM "queries_sqlpp/temporal/TemporalQueries.xml">
+ ]>
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp" SourceLocation="true">
+ &ObjectsQueries;
+ &AsyncDeferredQueries;
+ &GeoQueries;
+ &TemporalQueries;
+ <test-group name="api">
+ <test-case FilePath="api">
+ <compilation-unit name="compileonly">
+ <output-dir compare="Text">compileonly</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="multiple-param-values">
+ <output-dir compare="Text">multiple-param-values</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="readonly-request">
+ <output-dir compare="Text">readonly-request</output-dir>
+ <expected-error>ASX0044: CREATE_DATAVERSE statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: DATAVERSE_DROP statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: DATASET_DECL statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: DATASET_DROP statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: CREATE_INDEX statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: INDEX_DROP statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: INSERT statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: UPSERT statement is not supported in read-only mode</expected-error>
+ <expected-error>ASX0044: DELETE statement is not supported in read-only mode</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api" check-warnings="true">
+ <compilation-unit name="request-dataverse">
+ <output-dir compare="Text">request-dataverse</output-dir>
+ <expected-warn>ASX1063: Cannot find dataverse with name testUnknown</expected-warn>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="request-param-validation">
+ <output-dir compare="Text">request-param-validation</output-dir>
+ <expected-error>Invalid value for parameter 'format': foo</expected-error>
+ <expected-error>Invalid value for parameter 'pretty': bar</expected-error>
+ <expected-error>Invalid value for parameter 'plan-format': blah</expected-error>
+ <expected-error>Invalid value for parameter 'max-result-reads': foo</expected-error>
+ <expected-error>Invalid value for parameter 'max-result-reads': 9999999999999999999999999999999999999999</expected-error>
+ <expected-error>Invalid value for parameter 'max-warnings': baz</expected-error>
+ <expected-error>Invalid value for parameter 'max-warnings': 1.5</expected-error>
+ <expected-error>Invalid value for parameter 'mode': asyn</expected-error>
+ <expected-error>Invalid value for parameter 'timeout': 12</expected-error>
+ <expected-error>Invalid value for parameter 'args': 12</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Invalid value for parameter 'format': foo</expected-error>
+ <expected-error>Invalid value for parameter 'pretty': bar</expected-error>
+ <expected-error>Invalid value for parameter 'plan-format': blah</expected-error>
+ <expected-error>Invalid value for parameter 'max-result-reads': foo</expected-error>
+ <expected-error>Invalid value for parameter 'max-warnings': baz</expected-error>
+ <expected-error>Invalid value for parameter 'mode': asyn</expected-error>
+ <expected-error>Invalid value for parameter 'args': 12</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Invalid value for parameter 'profile': true</expected-error>
+ <expected-error>Invalid value for parameter 'profile': true</expected-error>
+ <expected-error>Invalid value for parameter 'profile': foo</expected-error>
+ <expected-error>Invalid value for parameter 'profile': foo</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="request-param-validation-400-BAD">
+ <output-dir compare="Text">request-param-validation-400-BAD</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="request-param">
+ <output-dir compare="Text">request-param</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-01">
+ <output-dir compare="Text">format-param-in-accept-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-02">
+ <output-dir compare="Clean-JSON">format-param-in-accept-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-03">
+ <output-dir compare="Lossless-JSON">format-param-in-accept-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-04">
+ <output-dir compare="Text">format-param-in-accept-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-05">
+ <output-dir compare="AST">format-param-in-accept-05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="get-non-query">
+ <output-dir compare="Text">get-non-query</output-dir>
+ <expected-error>CREATE_DATAVERSE statement is not supported in read-only mode</expected-error>
+ <expected-error>TYPE_DECL statement is not supported in read-only mode</expected-error>
+ <expected-error>CREATE_FUNCTION statement is not supported in read-only mode</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="get-query">
+ <output-dir compare="Text">get-query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="post-non-query">
+ <output-dir compare="Text">post-non-query</output-dir>
+ <expected-error>TYPE_DECL statement is not supported in read-only mode</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="signature">
+ <output-dir compare="Text">signature</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="ignore-body-for-get">
+ <output-dir compare="Text">ignore-body-for-get</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="flwor">
+ <test-case FilePath="flwor">
+ <compilation-unit name="at00">
+ <output-dir compare="Text">at00</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at01">
+ <output-dir compare="Text">at01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at02">
+ <output-dir compare="Text">at02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at03">
+ <output-dir compare="Text">at03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at04">
+ <output-dir compare="Text">at04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at05">
+ <output-dir compare="Text">at05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at06">
+ <output-dir compare="Text">at06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at07">
+ <output-dir compare="Text">at07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-13">
+ <output-dir compare="Text">order-by-13</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="query-issue550">
+ <output-dir compare="Text">query-issue550</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="query-ASTERIXDB-883">
+ <output-dir compare="Text">query-ASTERIXDB-883</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="query-ASTERIXDB-1576">
+ <output-dir compare="Text">query-ASTERIXDB-883</output-dir>
+ <expected-error>Duplicate alias definitions: samptable1</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="query-ASTERIXDB-1576-2">
+ <output-dir compare="Text">query-ASTERIXDB-883</output-dir>
+ <expected-error>Duplicate alias definitions: s2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="query-ASTERIXDB-1576-3">
+ <output-dir compare="Text">query-ASTERIXDB-883</output-dir>
+ <expected-error>Duplicate alias definitions: s1</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor" check-warnings="true">
+ <compilation-unit name="query-ASTERIXDB-2446">
+ <output-dir compare="Text">query-ASTERIXDB-2446</output-dir>
+ <expected-error>ASX0013: Duplicate field name 'a'</expected-error>
+ <expected-warn>Duplicate field name 'c' (in line 28, at column 84)</expected-warn>
+ <expected-warn>Duplicate field name 'e' (in line 28, at column 116)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="query-ASTERIXDB-2446-2">
+ <output-dir compare="Text">query-ASTERIXDB-883</output-dir>
+ <expected-error>ASX0013: Duplicate field name 'a' (in line 27, at column 20)</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'b' (in line 27, at column 20)</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'c' (in line 27, at column 11)</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'd' (in line 27, at column 11)</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'e' (in line 27, at column 14)</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'f' (in line 27, at column 11)</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'g' (in line 27, at column 11)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="select-let">
+ <output-dir compare="Text">select-let</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let33">
+ <output-dir compare="Text">let33</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="at00">
+ <output-dir compare="Text">at00</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="flwor">
+ <compilation-unit name="query-ASTERIXDB-1463">
+ <output-dir compare="Text">query-ASTERIXDB-1485</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="query-ASTERIXDB-1485">
+ <output-dir compare="Text">query-ASTERIXDB-1485</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ </test-group>
+ <test-group name="sorting">
+ <test-case FilePath="sorting">
+ <compilation-unit name="arrays">
+ <output-dir compare="Text">arrays</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sorting">
+ <compilation-unit name="range_hint">
+ <output-dir compare="Text">range_hint</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sorting">
+ <compilation-unit name="records">
+ <output-dir compare="Text">records</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="explain">
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_simple">
+ <output-dir compare="Text">explain_simple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_caps">
+ <output-dir compare="Text">explain_simple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_pretty">
+ <parameter name="pretty" value="true" />
+ <output-dir compare="Text">explain_simple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_object_constructor-01">
+ <output-dir compare="Text">explain_object_constructor-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_object_constructor-02">
+ <output-dir compare="Text">explain_object_constructor-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_object_constructor-03">
+ <output-dir compare="Text">explain_object_constructor-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_field_access">
+ <output-dir compare="Text">explain_field_access</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_field_access_closed">
+ <output-dir compare="Text">explain_field_access_closed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_negative">
+ <output-dir compare="Text">explain_simple</output-dir>
+ <expected-error>ASX1001: Syntax error: EXPLAIN is not supported for this kind of statement</expected-error>
+ <expected-error>ASX1001: Syntax error: EXPLAIN is not supported for this kind of statement</expected-error>
+ <expected-error>ASX1001: Syntax error: EXPLAIN is not supported for this kind of statement</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="aggregate">
+ <test-case FilePath="aggregate">
+ <compilation-unit name="issue531_string_min_max">
+ <output-dir compare="Text">issue531_string_min_max</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="agg_null">
+ <output-dir compare="Text">agg_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="agg_null_rec">
+ <output-dir compare="Text">agg_null_rec</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="agg_null_rec_1">
+ <output-dir compare="Text">agg_null_rec_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="agg_number_rec">
+ <output-dir compare="Text">agg_number_rec</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate" check-warnings="true">
+ <compilation-unit name="avg_mixed">
+ <output-dir compare="Text">avg_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-avg cannot process input type string (in line 26, at column 16)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate" check-warnings="true">
+ <compilation-unit name="serial_avg_mixed">
+ <output-dir compare="Text">serial_avg_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-avg cannot process input type string (in line 29, at column 39)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="group_only">
+ <output-dir compare="Text">group_only</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="min_mixed">
+ <output-dir compare="Text">min_mixed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_mixed">
+ <output-dir compare="Text">stddev_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-stddev_samp gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_mixed">
+ <output-dir compare="Text">serial_stddev_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-stddev_samp gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_pop_mixed">
+ <output-dir compare="Text">stddev_pop_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-stddev_pop gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_pop_mixed">
+ <output-dir compare="Text">serial_stddev_pop_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-stddev_pop gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate" check-warnings="true">
+ <compilation-unit name="sum/sum_mixed">
+ <output-dir compare="Text">sum/sum_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-sum cannot process input type string (in line 27, at column 16)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate" check-warnings="true">
+ <compilation-unit name="sum/serial_sum_mixed">
+ <output-dir compare="Text">sum/serial_sum_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-sum cannot process input type string (in line 29, at column 39)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_mixed">
+ <output-dir compare="Text">var_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-var_samp gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_mixed">
+ <output-dir compare="Text">serial_var_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-var_samp gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_pop_mixed">
+ <output-dir compare="Text">var_pop_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-var_pop gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_pop_mixed">
+ <output-dir compare="Text">serial_var_pop_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-var_pop gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_mixed">
+ <output-dir compare="Text">kurtosis_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-kurtosis gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_mixed">
+ <output-dir compare="Text">serial_kurtosis_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-kurtosis gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_mixed">
+ <output-dir compare="Text">skewness_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-skewness gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_mixed">
+ <output-dir compare="Text">serial_skewness_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-skewness gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="agg_number">
+ <output-dir compare="Text">agg_number</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="issue425_min_hetero_list_1">
+ <output-dir compare="Text">issue425_min_hetero_list_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="issue425_min_hetero_list">
+ <output-dir compare="Text">issue425_min_hetero_list</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/issue425_sum_hetero_list_1">
+ <output-dir compare="Text">sum/issue425_sum_hetero_list_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/issue425_sum_hetero_list">
+ <output-dir compare="Text">sum/issue425_sum_hetero_list</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="query-issue400">
+ <output-dir compare="Text">query-issue400</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="issue395">
+ <output-dir compare="Text">issue395</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="issue412_0">
+ <output-dir compare="Text">issue412_0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="issue412_1">
+ <output-dir compare="Text">issue412_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_double">
+ <output-dir compare="Text">avg_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_double_null">
+ <output-dir compare="Text">avg_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_empty_01">
+ <output-dir compare="Text">avg_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_empty_02">
+ <output-dir compare="Text">avg_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_float">
+ <output-dir compare="Text">avg_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_float_null">
+ <output-dir compare="Text">avg_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int16">
+ <output-dir compare="Text">avg_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int16_null">
+ <output-dir compare="Text">avg_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int32">
+ <output-dir compare="Text">avg_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int32_null">
+ <output-dir compare="Text">avg_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int64">
+ <output-dir compare="Text">avg_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int64_null">
+ <output-dir compare="Text">avg_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int8">
+ <output-dir compare="Text">avg_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_int8_null">
+ <output-dir compare="Text">avg_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="avg_distinct">
+ <output-dir compare="Text">avg_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="count_01">
+ <output-dir compare="Text">count_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="count_dataset">
+ <output-dir compare="Text">count_dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="count_empty_01">
+ <output-dir compare="Text">count_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="count_empty_02">
+ <output-dir compare="Text">count_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="count_null">
+ <output-dir compare="Text">count_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="count_distinct">
+ <output-dir compare="Text">count_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_double">
+ <output-dir compare="Text">kurtosis_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_double_null">
+ <output-dir compare="Text">kurtosis_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_empty_01">
+ <output-dir compare="Text">kurtosis_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_empty_02">
+ <output-dir compare="Text">kurtosis_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_float">
+ <output-dir compare="Text">kurtosis_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_float_null">
+ <output-dir compare="Text">kurtosis_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int16">
+ <output-dir compare="Text">kurtosis_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int16_null">
+ <output-dir compare="Text">kurtosis_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int32">
+ <output-dir compare="Text">kurtosis_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int32_null">
+ <output-dir compare="Text">kurtosis_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int64">
+ <output-dir compare="Text">kurtosis_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int64_null">
+ <output-dir compare="Text">kurtosis_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int8">
+ <output-dir compare="Text">kurtosis_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_int8_null">
+ <output-dir compare="Text">kurtosis_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="kurtosis_distinct">
+ <output-dir compare="Text">kurtosis_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="max_empty_01">
+ <output-dir compare="Text">max_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="max_empty_02">
+ <output-dir compare="Text">max_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="max_distinct">
+ <output-dir compare="Text">max_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="min_empty_01">
+ <output-dir compare="Text">min_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="min_empty_02">
+ <output-dir compare="Text">min_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="min_distinct">
+ <output-dir compare="Text">min_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_avg">
+ <output-dir compare="Text">scalar_avg</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_avg_empty">
+ <output-dir compare="Text">scalar_avg_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_avg_null">
+ <output-dir compare="Text">scalar_avg_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_count">
+ <output-dir compare="Text">scalar_count</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_count_empty">
+ <output-dir compare="Text">scalar_count_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_count_null">
+ <output-dir compare="Text">scalar_count_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_kurtosis">
+ <output-dir compare="Text">scalar_kurtosis</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_kurtosis_empty">
+ <output-dir compare="Text">scalar_kurtosis_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_kurtosis_null">
+ <output-dir compare="Text">scalar_kurtosis_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_max">
+ <output-dir compare="Text">scalar_max</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_max_empty">
+ <output-dir compare="Text">scalar_max_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_max_null">
+ <output-dir compare="Text">scalar_max_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_min">
+ <output-dir compare="Text">scalar_min</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_min_empty">
+ <output-dir compare="Text">scalar_min_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_min_null">
+ <output-dir compare="Text">scalar_min_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_skewness">
+ <output-dir compare="Text">scalar_skewness</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_skewness_empty">
+ <output-dir compare="Text">scalar_skewness_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_skewness_null">
+ <output-dir compare="Text">scalar_skewness_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_stddev">
+ <output-dir compare="Text">scalar_stddev</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_stddev_empty">
+ <output-dir compare="Text">scalar_stddev_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_stddev_null">
+ <output-dir compare="Text">scalar_stddev_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/scalar_sum">
+ <output-dir compare="Text">sum/scalar_sum</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/scalar_sum_empty">
+ <output-dir compare="Text">sum/scalar_sum_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/scalar_sum_null">
+ <output-dir compare="Text">sum/scalar_sum_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/scalar_sum_type">
+ <output-dir compare="Text">sum/scalar_sum_type</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_var">
+ <output-dir compare="Text">scalar_var</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_var_empty">
+ <output-dir compare="Text">scalar_var_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="scalar_var_null">
+ <output-dir compare="Text">scalar_var_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_empty">
+ <output-dir compare="Text">serial_avg_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int8">
+ <output-dir compare="Text">serial_avg_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int8_null">
+ <output-dir compare="Text">serial_avg_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int16">
+ <output-dir compare="Text">serial_avg_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int16_null">
+ <output-dir compare="Text">serial_avg_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int32">
+ <output-dir compare="Text">serial_avg_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int32_null">
+ <output-dir compare="Text">serial_avg_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int64">
+ <output-dir compare="Text">serial_avg_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_int64_null">
+ <output-dir compare="Text">serial_avg_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_float">
+ <output-dir compare="Text">serial_avg_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_float_null">
+ <output-dir compare="Text">serial_avg_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_double">
+ <output-dir compare="Text">serial_avg_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_avg_double_null">
+ <output-dir compare="Text">serial_avg_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_double">
+ <output-dir compare="Text">serial_kurtosis_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_double_null">
+ <output-dir compare="Text">serial_kurtosis_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_empty">
+ <output-dir compare="Text">serial_kurtosis_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_float">
+ <output-dir compare="Text">serial_kurtosis_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_float_null">
+ <output-dir compare="Text">serial_kurtosis_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int16">
+ <output-dir compare="Text">serial_kurtosis_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int16_null">
+ <output-dir compare="Text">serial_kurtosis_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int32">
+ <output-dir compare="Text">serial_kurtosis_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int32_null">
+ <output-dir compare="Text">serial_kurtosis_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int64">
+ <output-dir compare="Text">serial_kurtosis_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int64_null">
+ <output-dir compare="Text">serial_kurtosis_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int8">
+ <output-dir compare="Text">serial_kurtosis_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_kurtosis_int8_null">
+ <output-dir compare="Text">serial_kurtosis_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_double">
+ <output-dir compare="Text">serial_skewness_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_double_null">
+ <output-dir compare="Text">serial_skewness_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_empty">
+ <output-dir compare="Text">serial_skewness_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_float">
+ <output-dir compare="Text">serial_skewness_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_float_null">
+ <output-dir compare="Text">serial_skewness_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int16">
+ <output-dir compare="Text">serial_skewness_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int16_null">
+ <output-dir compare="Text">serial_skewness_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int32">
+ <output-dir compare="Text">serial_skewness_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int32_null">
+ <output-dir compare="Text">serial_skewness_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int64">
+ <output-dir compare="Text">serial_skewness_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int64_null">
+ <output-dir compare="Text">serial_skewness_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int8">
+ <output-dir compare="Text">serial_skewness_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_skewness_int8_null">
+ <output-dir compare="Text">serial_skewness_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_double">
+ <output-dir compare="Text">serial_stddev_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_double_null">
+ <output-dir compare="Text">serial_stddev_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_empty">
+ <output-dir compare="Text">serial_stddev_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_float">
+ <output-dir compare="Text">serial_stddev_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_float_null">
+ <output-dir compare="Text">serial_stddev_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int16">
+ <output-dir compare="Text">serial_stddev_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int16_null">
+ <output-dir compare="Text">serial_stddev_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int32">
+ <output-dir compare="Text">serial_stddev_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int32_null">
+ <output-dir compare="Text">serial_stddev_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int64">
+ <output-dir compare="Text">serial_stddev_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int64_null">
+ <output-dir compare="Text">serial_stddev_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int8">
+ <output-dir compare="Text">serial_stddev_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_stddev_int8_null">
+ <output-dir compare="Text">serial_stddev_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_double">
+ <output-dir compare="Text">sum/serial_sum_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_double_null">
+ <output-dir compare="Text">sum/serial_sum_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_empty">
+ <output-dir compare="Text">sum/serial_sum_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_float">
+ <output-dir compare="Text">sum/serial_sum_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_float_null">
+ <output-dir compare="Text">sum/serial_sum_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int16">
+ <output-dir compare="Text">sum/serial_sum_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int16_null">
+ <output-dir compare="Text">sum/serial_sum_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int32">
+ <output-dir compare="Text">sum/serial_sum_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int32_null">
+ <output-dir compare="Text">sum/serial_sum_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int64">
+ <output-dir compare="Text">sum/serial_sum_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int64_null">
+ <output-dir compare="Text">sum/serial_sum_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int64_overflow">
+ <output-dir compare="Text">sum/serial_sum_int64_overflow</output-dir>
+ <expected-error>Overflow in agg-sum</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int8">
+ <output-dir compare="Text">sum/serial_sum_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/serial_sum_int8_null">
+ <output-dir compare="Text">sum/serial_sum_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_double">
+ <output-dir compare="Text">serial_var_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_double_null">
+ <output-dir compare="Text">serial_var_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_empty">
+ <output-dir compare="Text">serial_var_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_float">
+ <output-dir compare="Text">serial_var_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_float_null">
+ <output-dir compare="Text">serial_var_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int16">
+ <output-dir compare="Text">serial_var_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int16_null">
+ <output-dir compare="Text">serial_var_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int32">
+ <output-dir compare="Text">serial_var_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int32_null">
+ <output-dir compare="Text">serial_var_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int64">
+ <output-dir compare="Text">serial_var_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int64_null">
+ <output-dir compare="Text">serial_var_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int8">
+ <output-dir compare="Text">serial_var_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="serial_var_int8_null">
+ <output-dir compare="Text">serial_var_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_double">
+ <output-dir compare="Text">skewness_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_double_null">
+ <output-dir compare="Text">skewness_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_empty_01">
+ <output-dir compare="Text">skewness_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_empty_02">
+ <output-dir compare="Text">skewness_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_float">
+ <output-dir compare="Text">skewness_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_float_null">
+ <output-dir compare="Text">skewness_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int16">
+ <output-dir compare="Text">skewness_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int16_null">
+ <output-dir compare="Text">skewness_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int32">
+ <output-dir compare="Text">skewness_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int32_null">
+ <output-dir compare="Text">skewness_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int64">
+ <output-dir compare="Text">skewness_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int64_null">
+ <output-dir compare="Text">skewness_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int8">
+ <output-dir compare="Text">skewness_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_int8_null">
+ <output-dir compare="Text">skewness_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="skewness_distinct">
+ <output-dir compare="Text">skewness_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_double">
+ <output-dir compare="Text">stddev_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_double_null">
+ <output-dir compare="Text">stddev_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_empty_01">
+ <output-dir compare="Text">stddev_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_empty_02">
+ <output-dir compare="Text">stddev_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_float">
+ <output-dir compare="Text">stddev_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_float_null">
+ <output-dir compare="Text">stddev_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int16">
+ <output-dir compare="Text">stddev_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int16_null">
+ <output-dir compare="Text">stddev_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int32">
+ <output-dir compare="Text">stddev_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int32_null">
+ <output-dir compare="Text">stddev_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int64">
+ <output-dir compare="Text">stddev_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int64_null">
+ <output-dir compare="Text">stddev_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int8">
+ <output-dir compare="Text">stddev_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_int8_null">
+ <output-dir compare="Text">stddev_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_distinct">
+ <output-dir compare="Text">stddev_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_misc">
+ <output-dir compare="Text">stddev_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="stddev_pop_misc">
+ <output-dir compare="Text">stddev_pop_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_double">
+ <output-dir compare="Text">sum/sum_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_double_null">
+ <output-dir compare="Text">sum/sum_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_empty_01">
+ <output-dir compare="Text">sum/sum_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_empty_02">
+ <output-dir compare="Text">sum/sum_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_float">
+ <output-dir compare="Text">sum/sum_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_float_null">
+ <output-dir compare="Text">sum/sum_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int16">
+ <output-dir compare="Text">sum/sum_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int16_null">
+ <output-dir compare="Text">sum/sum_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int32">
+ <output-dir compare="Text">sum/sum_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int32_null">
+ <output-dir compare="Text">sum/sum_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int64">
+ <output-dir compare="Text">sum/sum_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int64_null">
+ <output-dir compare="Text">sum/sum_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int64_null">
+ <output-dir compare="Text">sum/sum_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int64_overflow">
+ <output-dir compare="Text">sum/sum_int64_overflow</output-dir>
+ <expected-error>Overflow in agg-sum</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int8">
+ <output-dir compare="Text">sum/sum_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_int8_null">
+ <output-dir compare="Text">sum/sum_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_null-with-pred">
+ <output-dir compare="Text">sum/sum_null-with-pred</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_numeric_null">
+ <output-dir compare="Text">sum/sum_numeric_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="sum/sum_distinct">
+ <output-dir compare="Text">sum/sum_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_double">
+ <output-dir compare="Text">var_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_double_null">
+ <output-dir compare="Text">var_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_empty_01">
+ <output-dir compare="Text">var_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_empty_02">
+ <output-dir compare="Text">var_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_float">
+ <output-dir compare="Text">var_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_float_null">
+ <output-dir compare="Text">var_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int16">
+ <output-dir compare="Text">var_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int16_null">
+ <output-dir compare="Text">var_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int32">
+ <output-dir compare="Text">var_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int32_null">
+ <output-dir compare="Text">var_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int64">
+ <output-dir compare="Text">var_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int64_null">
+ <output-dir compare="Text">var_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int8">
+ <output-dir compare="Text">var_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_int8_null">
+ <output-dir compare="Text">var_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_distinct">
+ <output-dir compare="Text">var_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_misc">
+ <output-dir compare="Text">var_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="var_pop_misc">
+ <output-dir compare="Text">var_pop_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
+ <compilation-unit name="min_max_arrays">
+ <output-dir compare="Text">min_max_arrays</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="aggregate-sql">
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="min_max_arrays">
+ <output-dir compare="Text">min_max_arrays</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue531_string_min_max">
+ <output-dir compare="Text">issue531_string_min_max</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="agg_null">
+ <output-dir compare="Text">agg_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="agg_null_rec">
+ <output-dir compare="Text">agg_null_rec</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="agg_null_rec_1">
+ <output-dir compare="Text">agg_null_rec_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="agg_number_rec">
+ <output-dir compare="Text">agg_number_rec</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql" check-warnings="true">
+ <compilation-unit name="avg_mixed">
+ <output-dir compare="Text">avg_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-avg cannot process input type string (in line 26, at column 16)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql" check-warnings="true">
+ <compilation-unit name="serial_avg_mixed">
+ <output-dir compare="Text">serial_avg_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-avg cannot process input type string (in line 29, at column 38)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="min_mixed">
+ <output-dir compare="Text">min_mixed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_mixed">
+ <output-dir compare="Text">serial_stddev_mixed</output-dir>
+ <expected-error>Invalid item type: function agg-stddev_samp cannot process item type string in an input array (or multiset)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_mixed">
+ <output-dir compare="Text">stddev_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-stddev_samp gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_pop_mixed">
+ <output-dir compare="Text">serial_stddev_pop_mixed</output-dir>
+ <expected-error>Invalid item type: function agg-stddev_pop cannot process item type string in an input array (or multiset)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_pop_mixed">
+ <output-dir compare="Text">stddev_pop_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-stddev_pop gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql" check-warnings="true">
+ <compilation-unit name="sum/sum_mixed">
+ <output-dir compare="Text">sum/sum_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-sum cannot process input type string (in line 27, at column 16)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql" check-warnings="true">
+ <compilation-unit name="sum/serial_sum_mixed">
+ <output-dir compare="Text">sum/serial_sum_mixed</output-dir>
+ <expected-warn>Unsupported type: agg-sum cannot process input type string (in line 29, at column 38)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_mixed">
+ <output-dir compare="Text">serial_var_mixed</output-dir>
+ <expected-error>Invalid item type: function agg-var_samp cannot process item type string in an input array (or multiset)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_mixed">
+ <output-dir compare="Text">var_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-var_samp gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_pop_mixed">
+ <output-dir compare="Text">serial_var_pop_mixed</output-dir>
+ <expected-error>Invalid item type: function agg-var_pop cannot process item type string in an input array (or multiset)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_pop_mixed">
+ <output-dir compare="Text">var_pop_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-var_pop gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_mixed">
+ <output-dir compare="Text">serial_skewness_mixed</output-dir>
+ <expected-error>Invalid item type: function agg-skewness cannot process item type string in an input array (or multiset)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_mixed">
+ <output-dir compare="Text">skewness_mixed</output-dir>
+ <expected-error>Type incompatibility: function agg-skewness gets incompatible input values: string and float</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="agg_number">
+ <output-dir compare="Text">agg_number</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue425_min_hetero_list_1">
+ <output-dir compare="Text">issue425_min_hetero_list_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue425_min_hetero_list">
+ <output-dir compare="Text">issue425_min_hetero_list</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/issue425_sum_hetero_list_1">
+ <output-dir compare="Text">sum/issue425_sum_hetero_list_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/issue425_sum_hetero_list">
+ <output-dir compare="Text">sum/issue425_sum_hetero_list</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="query-issue400">
+ <output-dir compare="Text">query-issue400</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue395">
+ <output-dir compare="Text">issue395</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue412_0">
+ <output-dir compare="Text">issue412_0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue412_1">
+ <output-dir compare="Text">issue412_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue2348">
+ <output-dir compare="Text">issue2348</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="issue2411">
+ <output-dir compare="Text">issue2411</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_double">
+ <output-dir compare="Text">avg_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_double_null">
+ <output-dir compare="Text">avg_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_empty_01">
+ <output-dir compare="Text">avg_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_empty_02">
+ <output-dir compare="Text">avg_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_float">
+ <output-dir compare="Text">avg_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_float_null">
+ <output-dir compare="Text">avg_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int16">
+ <output-dir compare="Text">avg_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int16_null">
+ <output-dir compare="Text">avg_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int32">
+ <output-dir compare="Text">avg_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int32_null">
+ <output-dir compare="Text">avg_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int64">
+ <output-dir compare="Text">avg_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int64_null">
+ <output-dir compare="Text">avg_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int8">
+ <output-dir compare="Text">avg_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_int8_null">
+ <output-dir compare="Text">avg_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="avg_distinct">
+ <output-dir compare="Text">avg_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="count_01">
+ <output-dir compare="Text">count_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="count_dataset">
+ <output-dir compare="Text">count_dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="count_empty_01">
+ <output-dir compare="Text">count_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="count_empty_02">
+ <output-dir compare="Text">count_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="count_null">
+ <output-dir compare="Text">count_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="count_distinct">
+ <output-dir compare="Text">count_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_double">
+ <output-dir compare="Text">kurtosis_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_double_null">
+ <output-dir compare="Text">kurtosis_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_empty_01">
+ <output-dir compare="Text">kurtosis_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_empty_02">
+ <output-dir compare="Text">kurtosis_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_float">
+ <output-dir compare="Text">kurtosis_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_float_null">
+ <output-dir compare="Text">kurtosis_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int16">
+ <output-dir compare="Text">kurtosis_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int16_null">
+ <output-dir compare="Text">kurtosis_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int32">
+ <output-dir compare="Text">kurtosis_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int32_null">
+ <output-dir compare="Text">kurtosis_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int64">
+ <output-dir compare="Text">kurtosis_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int64_null">
+ <output-dir compare="Text">kurtosis_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int8">
+ <output-dir compare="Text">kurtosis_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_int8_null">
+ <output-dir compare="Text">kurtosis_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="kurtosis_distinct">
+ <output-dir compare="Text">kurtosis_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="max_empty_01">
+ <output-dir compare="Text">max_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="max_empty_02">
+ <output-dir compare="Text">max_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="max_distinct">
+ <output-dir compare="Text">max_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="min_empty_01">
+ <output-dir compare="Text">min_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="min_empty_02">
+ <output-dir compare="Text">min_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="min_distinct">
+ <output-dir compare="Text">min_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_avg">
+ <output-dir compare="Text">scalar_avg</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_avg_empty">
+ <output-dir compare="Text">scalar_avg_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_avg_null">
+ <output-dir compare="Text">scalar_avg_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_count">
+ <output-dir compare="Text">scalar_count</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_count_empty">
+ <output-dir compare="Text">scalar_count_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_count_null">
+ <output-dir compare="Text">scalar_count_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_kurtosis">
+ <output-dir compare="Text">scalar_kurtosis</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_kurtosis_empty">
+ <output-dir compare="Text">scalar_kurtosis_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_kurtosis_null">
+ <output-dir compare="Text">scalar_kurtosis_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_max">
+ <output-dir compare="Text">scalar_max</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_max_empty">
+ <output-dir compare="Text">scalar_max_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_max_null">
+ <output-dir compare="Text">scalar_max_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_min">
+ <output-dir compare="Text">scalar_min</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_min_empty">
+ <output-dir compare="Text">scalar_min_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_min_null">
+ <output-dir compare="Text">scalar_min_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_skewness">
+ <output-dir compare="Text">scalar_skewness</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_skewness_empty">
+ <output-dir compare="Text">scalar_skewness_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_skewness_null">
+ <output-dir compare="Text">scalar_skewness_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_stddev">
+ <output-dir compare="Text">scalar_stddev</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_stddev_empty">
+ <output-dir compare="Text">scalar_stddev_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_stddev_null">
+ <output-dir compare="Text">scalar_stddev_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/scalar_sum">
+ <output-dir compare="Text">sum/scalar_sum</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/scalar_sum_empty">
+ <output-dir compare="Text">sum/scalar_sum_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/scalar_sum_null">
+ <output-dir compare="Text">sum/scalar_sum_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/scalar_sum_type">
+ <output-dir compare="Text">sum/scalar_sum_type</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_var">
+ <output-dir compare="Text">scalar_var</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_var_empty">
+ <output-dir compare="Text">scalar_var_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="scalar_var_null">
+ <output-dir compare="Text">scalar_var_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_empty">
+ <output-dir compare="Text">serial_avg_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int8">
+ <output-dir compare="Text">serial_avg_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int8_null">
+ <output-dir compare="Text">serial_avg_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int16">
+ <output-dir compare="Text">serial_avg_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int16_null">
+ <output-dir compare="Text">serial_avg_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int32">
+ <output-dir compare="Text">serial_avg_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int32_null">
+ <output-dir compare="Text">serial_avg_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int64">
+ <output-dir compare="Text">serial_avg_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_int64_null">
+ <output-dir compare="Text">serial_avg_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_float">
+ <output-dir compare="Text">serial_avg_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_float_null">
+ <output-dir compare="Text">serial_avg_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_double">
+ <output-dir compare="Text">serial_avg_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_avg_double_null">
+ <output-dir compare="Text">serial_avg_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_double">
+ <output-dir compare="Text">serial_kurtosis_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_double_null">
+ <output-dir compare="Text">serial_kurtosis_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_empty">
+ <output-dir compare="Text">serial_kurtosis_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_float">
+ <output-dir compare="Text">serial_kurtosis_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_float_null">
+ <output-dir compare="Text">serial_kurtosis_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int16">
+ <output-dir compare="Text">serial_kurtosis_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int16_null">
+ <output-dir compare="Text">serial_kurtosis_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int32">
+ <output-dir compare="Text">serial_kurtosis_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int32_null">
+ <output-dir compare="Text">serial_kurtosis_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int64">
+ <output-dir compare="Text">serial_kurtosis_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int64_null">
+ <output-dir compare="Text">serial_kurtosis_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int8">
+ <output-dir compare="Text">serial_kurtosis_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_kurtosis_int8_null">
+ <output-dir compare="Text">serial_kurtosis_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_double">
+ <output-dir compare="Text">serial_skewness_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_double_null">
+ <output-dir compare="Text">serial_skewness_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_empty">
+ <output-dir compare="Text">serial_skewness_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_float">
+ <output-dir compare="Text">serial_skewness_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_float_null">
+ <output-dir compare="Text">serial_skewness_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int16">
+ <output-dir compare="Text">serial_skewness_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int16_null">
+ <output-dir compare="Text">serial_skewness_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int32">
+ <output-dir compare="Text">serial_skewness_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int32_null">
+ <output-dir compare="Text">serial_skewness_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int64">
+ <output-dir compare="Text">serial_skewness_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int64_null">
+ <output-dir compare="Text">serial_skewness_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int8">
+ <output-dir compare="Text">serial_skewness_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_skewness_int8_null">
+ <output-dir compare="Text">serial_skewness_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_double">
+ <output-dir compare="Text">serial_stddev_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_double_null">
+ <output-dir compare="Text">serial_stddev_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_empty">
+ <output-dir compare="Text">serial_stddev_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_float">
+ <output-dir compare="Text">serial_stddev_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_float_null">
+ <output-dir compare="Text">serial_stddev_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int16">
+ <output-dir compare="Text">serial_stddev_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int16_null">
+ <output-dir compare="Text">serial_stddev_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int32">
+ <output-dir compare="Text">serial_stddev_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int32_null">
+ <output-dir compare="Text">serial_stddev_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int64">
+ <output-dir compare="Text">serial_stddev_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int64_null">
+ <output-dir compare="Text">serial_stddev_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int8">
+ <output-dir compare="Text">serial_stddev_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_stddev_int8_null">
+ <output-dir compare="Text">serial_stddev_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_double">
+ <output-dir compare="Text">sum/serial_sum_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_double_null">
+ <output-dir compare="Text">sum/serial_sum_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_empty">
+ <output-dir compare="Text">sum/serial_sum_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_float">
+ <output-dir compare="Text">sum/serial_sum_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_float_null">
+ <output-dir compare="Text">sum/serial_sum_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int16">
+ <output-dir compare="Text">sum/serial_sum_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int16_null">
+ <output-dir compare="Text">sum/serial_sum_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int32">
+ <output-dir compare="Text">sum/serial_sum_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int32_null">
+ <output-dir compare="Text">sum/serial_sum_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int64">
+ <output-dir compare="Text">sum/serial_sum_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int64_null">
+ <output-dir compare="Text">sum/serial_sum_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int64_overflow">
+ <output-dir compare="Text">sum/serial_sum_int64_overflow</output-dir>
+ <expected-error>Overflow in agg-sum</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int8">
+ <output-dir compare="Text">sum/serial_sum_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/serial_sum_int8_null">
+ <output-dir compare="Text">sum/serial_sum_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_double">
+ <output-dir compare="Text">serial_var_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_double_null">
+ <output-dir compare="Text">serial_var_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_empty">
+ <output-dir compare="Text">serial_var_empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_float">
+ <output-dir compare="Text">serial_var_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_float_null">
+ <output-dir compare="Text">serial_var_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int16">
+ <output-dir compare="Text">serial_var_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int16_null">
+ <output-dir compare="Text">serial_var_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int32">
+ <output-dir compare="Text">serial_var_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int32_null">
+ <output-dir compare="Text">serial_var_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int64">
+ <output-dir compare="Text">serial_var_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int64_null">
+ <output-dir compare="Text">serial_var_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int8">
+ <output-dir compare="Text">serial_var_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="serial_var_int8_null">
+ <output-dir compare="Text">serial_var_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_double">
+ <output-dir compare="Text">skewness_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_double_null">
+ <output-dir compare="Text">skewness_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_empty_01">
+ <output-dir compare="Text">skewness_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_empty_02">
+ <output-dir compare="Text">skewness_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_float">
+ <output-dir compare="Text">skewness_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_float_null">
+ <output-dir compare="Text">skewness_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int16">
+ <output-dir compare="Text">skewness_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int16_null">
+ <output-dir compare="Text">skewness_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int32">
+ <output-dir compare="Text">skewness_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int32_null">
+ <output-dir compare="Text">skewness_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int64">
+ <output-dir compare="Text">skewness_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int64_null">
+ <output-dir compare="Text">skewness_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int8">
+ <output-dir compare="Text">skewness_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_int8_null">
+ <output-dir compare="Text">skewness_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="skewness_distinct">
+ <output-dir compare="Text">skewness_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_double">
+ <output-dir compare="Text">stddev_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_double_null">
+ <output-dir compare="Text">stddev_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_empty_01">
+ <output-dir compare="Text">stddev_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_empty_02">
+ <output-dir compare="Text">stddev_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_float">
+ <output-dir compare="Text">stddev_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_float_null">
+ <output-dir compare="Text">stddev_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int16">
+ <output-dir compare="Text">stddev_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int16_null">
+ <output-dir compare="Text">stddev_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int32">
+ <output-dir compare="Text">stddev_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int32_null">
+ <output-dir compare="Text">stddev_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int64">
+ <output-dir compare="Text">stddev_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int64_null">
+ <output-dir compare="Text">stddev_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int8">
+ <output-dir compare="Text">stddev_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_int8_null">
+ <output-dir compare="Text">stddev_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_distinct">
+ <output-dir compare="Text">stddev_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_misc">
+ <output-dir compare="Text">stddev_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="stddev_pop_misc">
+ <output-dir compare="Text">stddev_pop_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_double">
+ <output-dir compare="Text">sum/sum_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_double_null">
+ <output-dir compare="Text">sum/sum_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_empty_01">
+ <output-dir compare="Text">sum/sum_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_empty_02">
+ <output-dir compare="Text">sum/sum_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_float">
+ <output-dir compare="Text">sum/sum_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_float_null">
+ <output-dir compare="Text">sum/sum_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int16">
+ <output-dir compare="Text">sum/sum_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int16_null">
+ <output-dir compare="Text">sum/sum_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int32">
+ <output-dir compare="Text">sum/sum_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int32_null">
+ <output-dir compare="Text">sum/sum_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int64">
+ <output-dir compare="Text">sum/sum_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int64_null">
+ <output-dir compare="Text">sum/sum_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int64_overflow">
+ <output-dir compare="Text">sum/sum_int64_overflow</output-dir>
+ <expected-error>Overflow in agg-sum</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int8">
+ <output-dir compare="Text">sum/sum_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_int8_null">
+ <output-dir compare="Text">sum/sum_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_null-with-pred">
+ <output-dir compare="Text">sum/sum_null-with-pred</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_numeric_null">
+ <output-dir compare="Text">sum/sum_numeric_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="sum/sum_distinct">
+ <output-dir compare="Text">sum/sum_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_double">
+ <output-dir compare="Text">var_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_double_null">
+ <output-dir compare="Text">var_double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_empty_01">
+ <output-dir compare="Text">var_empty_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_empty_02">
+ <output-dir compare="Text">var_empty_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_float">
+ <output-dir compare="Text">var_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_float_null">
+ <output-dir compare="Text">var_float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int16">
+ <output-dir compare="Text">var_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int16_null">
+ <output-dir compare="Text">var_int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int32">
+ <output-dir compare="Text">var_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int32_null">
+ <output-dir compare="Text">var_int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int64">
+ <output-dir compare="Text">var_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int64_null">
+ <output-dir compare="Text">var_int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int8">
+ <output-dir compare="Text">var_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_int8_null">
+ <output-dir compare="Text">var_int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_distinct">
+ <output-dir compare="Text">var_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_misc">
+ <output-dir compare="Text">var_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql">
+ <compilation-unit name="var_pop_misc">
+ <output-dir compare="Text">var_pop_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="aggregate-sql-sugar">
+ <test-case FilePath="aggregate-sql-sugar">
+ <compilation-unit name="array_agg">
+ <output-dir compare="Text">array_agg</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql-sugar">
+ <compilation-unit name="array_agg_negative">
+ <output-dir compare="Text">array_agg</output-dir>
+ <expected-error>ASX1079: Compilation error: arrayagg is a SQL-92 aggregate function. The SQL++ core aggregate function strict_arrayagg could potentially express the intent.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql-sugar" check-warnings="true">
+ <compilation-unit name="avg_mixed">
+ <output-dir compare="Text">avg_mixed</output-dir>
+ <expected-warn>ASX0004: Unsupported type: agg-avg cannot process input type string (in line 26, at column 12)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: agg-avg cannot process input type string (in line 28, at column 19)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: agg-avg cannot process input type string (in line 29, at column 19)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql-sugar">
+ <compilation-unit name="distinct_mixed">
+ <output-dir compare="Text">distinct_mixed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql-sugar">
+ <compilation-unit name="stddev">
+ <output-dir compare="Text">stddev</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-sql-sugar">
+ <compilation-unit name="var">
+ <output-dir compare="Text">var</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="aggregate-subclause">
+ <test-case FilePath="aggregate-subclause">
+ <compilation-unit name="agg_filter_01">
+ <output-dir compare="Text">agg_filter_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-subclause">
+ <compilation-unit name="agg_filter_02_neg">
+ <output-dir compare="Text">agg_filter_01</output-dir>
+ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
+ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier r</expected-error>
+ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
+ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array_fun">
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_prepend">
+ <output-dir compare="Text">array_prepend</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_append">
+ <output-dir compare="Text">array_append</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_position">
+ <output-dir compare="Text">array_position</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_repeat">
+ <output-dir compare="Text">array_repeat</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_reverse">
+ <output-dir compare="Text">array_reverse</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_contains">
+ <output-dir compare="Text">array_contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_insert">
+ <output-dir compare="Text">array_insert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_put">
+ <output-dir compare="Text">array_put</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_remove">
+ <output-dir compare="Text">array_remove</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_distinct">
+ <output-dir compare="Text">array_distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_sort">
+ <output-dir compare="Text">array_sort</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_concat">
+ <output-dir compare="Text">array_concat</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_flatten">
+ <output-dir compare="Text">array_flatten</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_ifnull">
+ <output-dir compare="Text">array_ifnull</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_intersect">
+ <output-dir compare="Text">array_intersect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_range">
+ <output-dir compare="Text">array_range</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_replace">
+ <output-dir compare="Text">array_replace</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_star">
+ <output-dir compare="Text">array_star</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_symdiff">
+ <output-dir compare="Text">array_symdiff</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_symdiffn">
+ <output-dir compare="Text">array_symdiffn</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_union">
+ <output-dir compare="Text">array_union</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_slice/array_slice_double_argument">
+ <output-dir compare="Text">array_slice/array_slice_double_argument</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_slice/array_slice_int_argument">
+ <output-dir compare="Text">array_slice/array_slice_int_argument</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_slice/array_slice_negative_argument">
+ <output-dir compare="Text">array_slice/array_slice_negative_argument</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_slice/array_slice_missing_result">
+ <output-dir compare="Text">array_slice/array_slice_missing_result</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_slice/array_slice_null_result">
+ <output-dir compare="Text">array_slice/array_slice_null_result</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_slice/array_slice_exception_result">
+ <output-dir compare="Text">array_slice/array_slice_exception_result</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature TinySocial.array_slice()</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_except/001">
+ <output-dir compare="Text">array_except/001</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_except/002">
+ <output-dir compare="Text">array_except/002</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_except/003">
+ <output-dir compare="Text">array_except/003</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_except/004">
+ <output-dir compare="Text">array_except/004</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_except/005">
+ <output-dir compare="Text">array_except/005</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_except/006">
+ <output-dir compare="Text">array_except/006</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array_fun">
+ <compilation-unit name="array_errors">
+ <output-dir compare="Text">array_errors</output-dir>
+ <expected-error>Input contains different list types (in line 25, at column 8)</expected-error>
+ <expected-error>Input contains different list types (in line 25, at column 8)</expected-error>
+ <expected-error>Input contains different list types (in line 25, at column 8)</expected-error>
+ <expected-error>Input contains different list types (in line 25, at column 8)</expected-error>
+ <expected-error>Input contains different list types (in line 25, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="bitwise">
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_and_01">
+ <output-dir compare="Text">bit_and_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_and_02">
+ <output-dir compare="Text">bit_and_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_and_03">
+ <output-dir compare="Text">bit_and_03</output-dir>
+ <expected-error>Invalid number of arguments for function</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_and_04">
+ <output-dir compare="Text">bit_and_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_or_01">
+ <output-dir compare="Text">bit_or_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_or_02">
+ <output-dir compare="Text">bit_or_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_or_03">
+ <output-dir compare="Text">bit_or_03</output-dir>
+ <expected-error>Invalid number of arguments for function</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_or_04">
+ <output-dir compare="Text">bit_or_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_xor_01">
+ <output-dir compare="Text">bit_xor_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_xor_02">
+ <output-dir compare="Text">bit_xor_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_xor_03">
+ <output-dir compare="Text">bit_xor_03</output-dir>
+ <expected-error>Invalid number of arguments for function</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_xor_04">
+ <output-dir compare="Text">bit_xor_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_not_01">
+ <output-dir compare="Text">bit_not_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_not_02">
+ <output-dir compare="Text">bit_not_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_not_03">
+ <output-dir compare="Text">bit_not_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_count_01">
+ <output-dir compare="Text">bit_count_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_count_02">
+ <output-dir compare="Text">bit_count_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_count_03">
+ <output-dir compare="Text">bit_count_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_set_01">
+ <output-dir compare="Text">bit_set_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_set_02">
+ <output-dir compare="Text">bit_set_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_set_03">
+ <output-dir compare="Text">bit_set_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_clear_01">
+ <output-dir compare="Text">bit_clear_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_clear_02">
+ <output-dir compare="Text">bit_clear_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_clear_03">
+ <output-dir compare="Text">bit_clear_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_shift_01">
+ <output-dir compare="Text">bit_shift_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_shift_02">
+ <output-dir compare="Text">bit_shift_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_shift_03">
+ <output-dir compare="Text">bit_shift_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_shift_04">
+ <output-dir compare="Text">bit_shift_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_test_01">
+ <output-dir compare="Text">bit_test_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_test_02">
+ <output-dir compare="Text">bit_test_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_test_03">
+ <output-dir compare="Text">bit_test_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="bit_test_04">
+ <output-dir compare="Text">bit_test_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="is_bit_set_01">
+ <output-dir compare="Text">is_bit_set_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="is_bit_set_02">
+ <output-dir compare="Text">is_bit_set_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="bitwise">
+ <compilation-unit name="is_bit_set_03">
+ <output-dir compare="Text">is_bit_set_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="boolean">
+ <test-case FilePath="boolean">
+ <compilation-unit name="and_01">
+ <output-dir compare="Text">and_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="boolean">
+ <compilation-unit name="and_null">
+ <output-dir compare="Text">and_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="boolean">
+ <compilation-unit name="and_null_false">
+ <output-dir compare="Text">and_null_false</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="boolean">
+ <compilation-unit name="not_01">
+ <output-dir compare="Text">not_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="comparison">
+ <test-case FilePath="comparison">
+ <compilation-unit name="secondary_idx_lookup">
+ <output-dir compare="Text">secondary_idx_lookup</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="year_month_duration_order">
+ <output-dir compare="Text">year_month_duration_order</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="datetime_order">
+ <output-dir compare="Text">datetime_order</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="datetime_range">
+ <output-dir compare="Text">datetime_range</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="datetime_range_between">
+ <output-dir compare="Text">datetime_range</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="datetime_tzeq">
+ <output-dir compare="Text">datetime_tzeq</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="double">
+ <output-dir compare="Text">double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="double_gte_01">
+ <output-dir compare="Text">double_gte_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="double_missing">
+ <output-dir compare="Text">double_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="double_null">
+ <output-dir compare="Text">double_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="eq_01">
+ <output-dir compare="Text">eq_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="float">
+ <output-dir compare="Text">float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="float_missing">
+ <output-dir compare="Text">float_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="float_null">
+ <output-dir compare="Text">float_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="gt_01">
+ <output-dir compare="Text">gt_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="gte_01">
+ <output-dir compare="Text">gte_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="incompatible">
+ <output-dir compare="Text">incompatible</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int16">
+ <output-dir compare="Text">int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int16_missing">
+ <output-dir compare="Text">int16_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int16_null">
+ <output-dir compare="Text">int16_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int32">
+ <output-dir compare="Text">int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int32_missing">
+ <output-dir compare="Text">int32_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int32_null">
+ <output-dir compare="Text">int32_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int64">
+ <output-dir compare="Text">int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int64_missing">
+ <output-dir compare="Text">int64_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int64_null">
+ <output-dir compare="Text">int64_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int8">
+ <output-dir compare="Text">int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int8_missing">
+ <output-dir compare="Text">int8_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int8_null">
+ <output-dir compare="Text">int8_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="int_not_between">
+ <output-dir compare="Text">int_not_between</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="is_distinct_01">
+ <output-dir compare="Text">is_distinct_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="like">
+ <output-dir compare="Text">like</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="lt_01">
+ <output-dir compare="Text">lt_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="lte_01">
+ <output-dir compare="Text">lte_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="neq_01">
+ <output-dir compare="Text">neq_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="neq_02">
+ <output-dir compare="Text">neq_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="numeric-comparison_01">
+ <output-dir compare="Text">numeric-comparison_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="string">
+ <output-dir compare="Text">string</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="string_missing">
+ <output-dir compare="Text">string_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="string_null">
+ <output-dir compare="Text">string_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_equality">
+ <output-dir compare="Text">issue363_equality</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_inequality_duration">
+ <output-dir compare="Text">issue363_inequality_duration</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_inequality_interval">
+ <output-dir compare="Text">issue363_inequality_interval</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_inequality_point">
+ <output-dir compare="Text">issue363_inequality_point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_inequality_line">
+ <output-dir compare="Text">issue363_inequality_line</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_inequality_polygon">
+ <output-dir compare="Text">issue363_inequality_polygon</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_inequality_rectangle">
+ <output-dir compare="Text">issue363_inequality_rectangle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="issue363_inequality_circle">
+ <output-dir compare="Text">issue363_inequality_circle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="binary">
+ <output-dir compare="Text">binary</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="binary_null">
+ <output-dir compare="Text">binary_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="greatest_mixed">
+ <output-dir compare="Text">greatest_mixed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="least_mixed">
+ <output-dir compare="Text">least_mixed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="missingif">
+ <output-dir compare="Text">missingif</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="nullif">
+ <output-dir compare="Text">nullif</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="nanif">
+ <output-dir compare="Text">nanif</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="posinfif">
+ <output-dir compare="Text">posinfif</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="neginfif">
+ <output-dir compare="Text">neginfif</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="arrays">
+ <output-dir compare="Text">arrays</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison">
+ <compilation-unit name="circle-point">
+ <output-dir compare="Text">circle-point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="comparison" check-warnings="true">
+ <compilation-unit name="incomparable_types">
+ <output-dir compare="Text">incomparable_types</output-dir>
+ <expected-warn>Incomparable input types: string and bigint (in line 26, at column 13)</expected-warn>
+ <expected-warn>Incomparable input types: array and bigint (in line 23, at column 7)</expected-warn>
+ <expected-warn>Incomparable input types: point and point (in line 24, at column 18)</expected-warn>
+ <expected-warn>Incomparable input types: bigint and string (in line 25, at column 46)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="constructor">
+ <test-case FilePath="constructor">
+ <compilation-unit name="binary_01">
+ <output-dir compare="Text">binary_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="binary_02">
+ <output-dir compare="Text">binary_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for binary in @#!1 (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: hex() cannot process input type date (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: hex() cannot process input type array (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: hex() cannot process input type object (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for binary in @#!2 (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: base64() cannot process input type date (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: base64() cannot process input type array (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: base64() cannot process input type object (in line 27, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="add-null">
+ <output-dir compare="Text">add-null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="boolean_01">
+ <output-dir compare="Text">boolean_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="boolean_02">
+ <output-dir compare="Text">boolean_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for boolean in FALSE (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for boolean in TRUE (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: boolean() cannot process input type date (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: boolean() cannot process input type array (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: boolean() cannot process input type object (in line 28, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="circle_01">
+ <output-dir compare="Text">circle_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="date_01">
+ <output-dir compare="Text">date_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="date_02">
+ <output-dir compare="Text">date_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for date in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: date() cannot process input type boolean (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: date() cannot process input type bigint (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: date() cannot process input type time (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: date() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: date() cannot process input type object (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for date in 12/31/2020 (in line 30, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="datetime_01">
+ <output-dir compare="Text">datetime_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="datetime_02">
+ <output-dir compare="Text">datetime_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for datetime in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: datetime() cannot process input type boolean (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: datetime() cannot process input type bigint (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: datetime() cannot process input type array (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: datetime() cannot process input type object (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for datetime in 1951-12-27T12:20:15Z (in line 29, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="double_01">
+ <output-dir compare="Text">double_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="double_02">
+ <output-dir compare="Text">double_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for double in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: double() cannot process input type datetime (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: double() cannot process input type date (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: double() cannot process input type time (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: double() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: double() cannot process input type object (in line 29, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="duration_01">
+ <output-dir compare="Text">duration_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="duration_02">
+ <output-dir compare="Text">duration_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for duration in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: duration() cannot process input type bigint (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: duration() cannot process input type datetime (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: duration() cannot process input type date (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: duration() cannot process input type time (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: duration() cannot process input type array (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: duration() cannot process input type object (in line 30, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for yearmonthduration in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: year-month-duration() cannot process input type bigint (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: year-month-duration() cannot process input type datetime (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: year-month-duration() cannot process input type date (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: year-month-duration() cannot process input type time (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: year-month-duration() cannot process input type array (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: year-month-duration() cannot process input type object (in line 30, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for daytimeduration in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: day-time-duration() cannot process input type bigint (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: day-time-duration() cannot process input type datetime (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: day-time-duration() cannot process input type date (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: day-time-duration() cannot process input type time (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: day-time-duration() cannot process input type array (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: day-time-duration() cannot process input type object (in line 30, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="float_01">
+ <output-dir compare="Text">float_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="float_02">
+ <output-dir compare="Text">float_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for float in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: float() cannot process input type datetime (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: float() cannot process input type date (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: float() cannot process input type time (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: float() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: float() cannot process input type object (in line 29, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="int_01">
+ <output-dir compare="Text">int_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="int_02">
+ <output-dir compare="Text">int_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for tinyint in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int8() cannot process input type datetime (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int8() cannot process input type date (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int8() cannot process input type time (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int8() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int8() cannot process input type object (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for smallint in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int16() cannot process input type datetime (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int16() cannot process input type date (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int16() cannot process input type time (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int16() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int16() cannot process input type object (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for integer in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int32() cannot process input type datetime (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int32() cannot process input type date (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int32() cannot process input type time (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int32() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int32() cannot process input type object (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for bigint in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int64() cannot process input type datetime (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int64() cannot process input type date (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int64() cannot process input type time (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int64() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int64() cannot process input type object (in line 29, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="interval">
+ <output-dir compare="Text">interval</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="line_01">
+ <output-dir compare="Text">line_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="rectangle_01">
+ <output-dir compare="Text">rectangle_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="point_01">
+ <output-dir compare="Text">point_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="polygon_01">
+ <output-dir compare="Text">polygon_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="primitive-01">
+ <output-dir compare="Text">primitive-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="primitive-02">
+ <output-dir compare="Text">primitive-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="primitive-03">
+ <output-dir compare="Text">primitive-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="primitive-04">
+ <output-dir compare="Text">primitive-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="string_01">
+ <output-dir compare="Text">string_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="string_02">
+ <output-dir compare="Text">string_02</output-dir>
+ <expected-warn>ASX0004: Unsupported type: string() cannot process input type array (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: string() cannot process input type object (in line 25, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="time_01">
+ <output-dir compare="Text">time_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor" check-warnings="true">
+ <compilation-unit name="time_02">
+ <output-dir compare="Text">time_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for time in @#! (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: time() cannot process input type boolean (in line 25, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: time() cannot process input type bigint (in line 26, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: time() cannot process input type date (in line 27, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: time() cannot process input type array (in line 28, at column 13)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: time() cannot process input type object (in line 29, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for time in 11:58:59 (in line 30, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor/uuid">
+ <compilation-unit name="uuid_01">
+ <output-dir compare="Text">uuid_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor/uuid" check-warnings="true">
+ <compilation-unit name="uuid_02">
+ <output-dir compare="Text">uuid_02</output-dir>
+ <expected-warn>ASX0006: Invalid format for uuid in 02a199ca-bf58-412e-bd9f-60a0c975a8a- (in line 24, at column 13)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for uuid in 12345 (in line 25, at column 13)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="constructor">
+ <compilation-unit name="polygon-from-open-list_issue1627">
+ <output-dir compare="Text">polygon-from-open-list_issue1627</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="custord">
+ <!--
+ <test-case FilePath="custord">
+ <compilation-unit name="co">
+ <output-dir compare="Text">co</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_01">
+ <output-dir compare="Text">customer_q_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_02">
+ <output-dir compare="Text">customer_q_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_03">
+ <output-dir compare="Text">customer_q_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_04">
+ <output-dir compare="Text">customer_q_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_05">
+ <output-dir compare="Text">customer_q_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_06">
+ <output-dir compare="Text">customer_q_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_07">
+ <output-dir compare="Text">customer_q_07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="customer_q_08">
+ <output-dir compare="Text">customer_q_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="custord">
+ <compilation-unit name="denorm-cust-order_01">
+ <output-dir compare="Text">denorm-cust-order_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="custord">
+ <compilation-unit name="denorm-cust-order_02">
+ <output-dir compare="Text">denorm-cust-order_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="custord">
+ <compilation-unit name="denorm-cust-order_03">
+ <output-dir compare="Text">denorm-cust-order_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="custord">
+ <compilation-unit name="freq-clerk">
+ <output-dir compare="Text">freq-clerk</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_01">
+ <output-dir compare="Text">join_q_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_02">
+ <output-dir compare="Text">join_q_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_03">
+ <output-dir compare="Text">join_q_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_04">
+ <output-dir compare="Text">join_q_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_05">
+ <output-dir compare="Text">join_q_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_06">
+ <output-dir compare="Text">join_q_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_07">
+ <output-dir compare="Text">join_q_06</output-dir>
+ <expected-error>Cannot find dataset c in dataverse test nor an alias with name c</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_08">
+ <output-dir compare="Text">join_q_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_09">
+ <output-dir compare="Text">join_q_01</output-dir>
+ <expected-error>Cannot resolve ambiguous alias reference for identifier age</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="query-ASTERIXDB-1754">
+ <output-dir compare="Text">query-ASTERIXDB-1754</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="load-test">
+ <output-dir compare="Text">load-test</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="order_q_01">
+ <output-dir compare="Text">order_q_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="order_q_02">
+ <output-dir compare="Text">order_q_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="order_q_03">
+ <output-dir compare="Text">order_q_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="order_q_04">
+ <output-dir compare="Text">order_q_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="order_q_05">
+ <output-dir compare="Text">order_q_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="order_q_06">
+ <output-dir compare="Text">order_q_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="dapd">
+ <test-case FilePath="dapd">
+ <compilation-unit name="q1">
+ <output-dir compare="Text">q1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-2">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-2-negative">
+ <output-dir compare="Text">q2</output-dir>
+ <expected-error>Cannot find dataset e in dataverse test nor an alias with name e</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-3">
+ <output-dir compare="Text">q2</output-dir>
+ <expected-error>Cannot resolve ambiguous alias reference for identifier sig_id</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-4">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-5">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-6">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-7">
+ <output-dir compare="Text">q2-7</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-8">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-9">
+ <output-dir compare="Text">q2-9</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-10">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-11">
+ <output-dir compare="Text">q2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dapd">
+ <compilation-unit name="q2-12">
+ <output-dir compare="Text">q2-12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="dapd">
+ <compilation-unit name="q3">
+ <output-dir compare="Text">q3</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ </test-group>
+ <test-group name="ddl">
+ <test-case FilePath="ddl/create-index">
+ <compilation-unit name="create-index-1">
+ <output-dir compare="Text">create-index-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl/create-index">
+ <compilation-unit name="create-index-2">
+ <output-dir compare="Text">create-index-2</output-dir>
+ <expected-error>Syntax error: In line 53 >>create primary index sec_primary_idx1 on LineItem type rtree;<< Encountered "rtree" at column 58.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl/create-index">
+ <compilation-unit name="create-index-3">
+ <output-dir compare="Text">create-index-3</output-dir>
+ <expected-error>Syntax error: In line 53 >>create primary sec_primary_idx1 on LineItem;<< Encountered <IDENTIFIER> "sec_primary_idx1" at column 18.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl/create-index">
+ <compilation-unit name="create-index-4">
+ <output-dir compare="Text">create-index-4</output-dir>
+ <expected-error>Syntax error: In line 53 >>create primary index if not exists sec_primary_idx1 if not exists on LineItem;<< Encountered <IDENTIFIER> "sec_primary_idx1" at column 37.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl/create-index">
+ <compilation-unit name="create-index-5">
+ <output-dir compare="Text">create-index-5</output-dir>
+ <expected-error>Syntax error: In line 53 >>create primary index if exists sec_primary_idx1 on LineItem;<< Encountered "exists" at column 26.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl/create-index">
+ <compilation-unit name="create-index-6">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1050: Cannot find dataset with name LineItemView1 in dataverse test (in line 55, at column 1)</expected-error>
+ <expected-error>ASX1050: Cannot find dataset with name LineItemView2 in dataverse test (in line 60, at column 1)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl/create-index">
+ <compilation-unit name="create-inverted-index-with-variable-length-primary-key">
+ <output-dir compare="Text">create-inverted-index-with-variable-length-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="bad-type-ddl">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1079: Compilation error: Reserved type name $x</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'c' (in line 29, at column 19)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="create-dataset-inline-type-1">
+ <output-dir compare="Text">create-dataset-inline-type-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="create-dataset-inline-type-2">
+ <output-dir compare="Text">create-dataset-inline-type-2</output-dir>
+ <expected-error>ASX1082: Cannot find datatype with name test.$d$t$i$Cust1</expected-error>
+ <expected-error>ASX1082: Cannot find datatype with name test.$d$t$i$Cust2</expected-error>
+ <expected-error>ASX1082: Cannot find datatype with name my_unknown_type</expected-error>
+ <expected-error>ASX0013: Duplicate field name 'c_name' (in line 25, at column 22)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="drop-primary-index">
+ <output-dir compare="Text">drop-primary-index</output-dir>
+ <expected-error>Cannot drop index 'ds'. Drop dataset 'ds' to remove this index</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl" check-warnings="true">
+ <compilation-unit name="invalid-dataverse">
+ <output-dir compare="Text">invalid-dataverse</output-dir>
+ <source-location>false</source-location>
+ <expected-warn>Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-warn>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 27, at column 1)</expected-error>
+ <expected-warn>Cannot find dataverse with name fakeDataverse (in line 29, at column 1)</expected-warn>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 30, at column 1)</expected-error>
+ <expected-error>Cannot find datatype with name fakeDataverse.myType</expected-error>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 30, at column 1)</expected-error>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 32, at column 1)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-dataverse-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' invalid'</expected-error>
+ <expected-error>ASX1079: Compilation error: Invalid operation - Cannot create dataverse: asterix</expected-error>
+ <expected-error>ASX1079: Compilation error: Invalid operation - Cannot create dataverse: algebricks</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: '' (in line 24, at column 16)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 27, at column 16)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-dataset-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' invalid'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 29, at column 16)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'c/d' (in line 31, at column 14)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-feed-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' invalid'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 34, at column 13)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'c/d' (in line 42, at column 11)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-feed-policy-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'c/d' (in line 32, at column 23)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-index-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' invalid'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 29, at column 19)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'c/d' (in line 33, at column 12)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-nodegroup-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-type-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' invalid'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 29, at column 13)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'c/d' (in line 33, at column 11)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-udf-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: ''</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' invalid'</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 29, at column 17)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'c/d' (in line 33, at column 15)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-view-name">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1115: Invalid name for a database object: '' (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' a' (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: ' invalid' (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'a/b' (in line 29, at column 13)</expected-error>
+ <expected-error>ASX1115: Invalid name for a database object: 'c/d' (in line 32, at column 11)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="dataset-and-index-same-dataverse">
+ <output-dir compare="Text">dataset-and-index-same-dataverse</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl" check-warnings="true">
+ <compilation-unit name="drop_dataset_invalid_dataverse">
+ <output-dir compare="Text">drop_dataset_invalid_dataverse</output-dir>
+ <expected-error>ASX1063: Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-error>
+ <expected-error>ASX1063: Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-error>
+ <expected-error>ASX1063: Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-error>
+ <expected-error>ASX1063: Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-error>
+ <expected-error>ASX1050: Cannot find dataset with name fakeDataset1 in dataverse realDataverse (in line 22, at column 1)</expected-error>
+ <expected-warn>ASX1063: Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-warn>
+ <expected-warn>ASX1063: Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-warn>
+ <expected-warn>ASX1063: Cannot find dataverse with name fakeDataverse (in line 22, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="create_dataset_with_filter_on_meta">
+ <output-dir compare="Text">create_dataset_with_filter_on_meta</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="index-bad-fields">
+ <output-dir compare="Text">index-bad-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="index-cast-null">
+ <placeholder name="with" value="" />
+ <output-dir compare="Text">index-cast-null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="index-cast-null">
+ <placeholder name="with" value="WITH {'merge-policy': {'name': 'correlated-prefix','parameters': { 'max-mergable-component-size': 16384, 'max-tolerance-component-count': 3 }}}" />
+ <output-dir compare="Text">index-cast-null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="index-cast-null-negative">
+ <output-dir compare="Text">index-cast-null-negative</output-dir>
+ <expected-error>CAST modifier is only allowed for B-Tree indexes</expected-error>
+ <expected-error>CAST modifier cannot be specified together with ENFORCED</expected-error>
+ <expected-error>CAST modifier is used without specifying the type of the indexed field</expected-error>
+ <expected-error>Typed index on 'typed_f2' field could be created only for open datatype</expected-error>
+ <expected-error>Parameter invalid_date cannot be set</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="dml">
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-with-autogenerated-pk_adm-with-sec-primary-index">
+ <output-dir compare="Text">insert-with-autogenerated-pk_adm-with-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-returning-fieldname-qualified">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="compact-dataset-and-its-indexes">
+ <output-dir compare="Text">compact-dataset-and-its-indexes</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-constant-merge-policy">
+ <output-dir compare="Text">using-constant-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-prefix-merge-policy">
+ <output-dir compare="Text">using-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-concurrent-merge-policy">
+ <output-dir compare="Text">using-concurrent-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-correlated-prefix-merge-policy">
+ <output-dir compare="Text">using-correlated-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-correlated-prefix-merge-policy-with-feed">
+ <output-dir compare="Text">using-correlated-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="using-no-merge-policy">
+ <output-dir compare="Text">using-no-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-issue382">
+ <output-dir compare="Text">query-issue382</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-issue433">
+ <output-dir compare="Text">query-issue433</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-issue288">
+ <output-dir compare="Text">query-issue288</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-issue205">
+ <output-dir compare="Text">query-issue205</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="delete-from-loaded-dataset-with-index">
+ <output-dir compare="Text">delete-from-loaded-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="delete-from-loaded-dataset">
+ <output-dir compare="Text">delete-from-loaded-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="delete-syntax-change">
+ <output-dir compare="Text">delete-syntax-change</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="drop-empty-secondary-indexes">
+ <output-dir compare="Text">drop-empty-secondary-indexes</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="drop-index">
+ <output-dir compare="Text">drop-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="create-drop-cltype">
+ <output-dir compare="Text">create-drop-cltype</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="create-drop-opntype">
+ <output-dir compare="Text">create-drop-opntype</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="empty-load-with-index">
+ <output-dir compare="Text">empty-load-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-into-empty-dataset">
+ <output-dir compare="Text">insert-into-empty-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="dml">
+ <compilation-unit name="insert-into-empty-dataset-with-index">
+ <output-dir compare="Text">insert-into-empty-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-syntax">
+ <output-dir compare="Text">insert-syntax</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-and-scan-dataset">
+ <output-dir compare="Text">insert-and-scan-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-and-scan-dataset-with-index">
+ <output-dir compare="Text">insert-and-scan-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="recreate-index">
+ <output-dir compare="Text">recreate-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-and-scan-joined-datasets">
+ <output-dir compare="Text">insert-and-scan-joined-datasets</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="dml">
+ <compilation-unit name="insert-into-loaded-dataset-with-index_01">
+ <output-dir compare="Text">insert-into-loaded-dataset-with-index_01</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <!--test-case FilePath="dml">
+ <compilation-unit name="insert-into-loaded-dataset-with-index_02">
+ <output-dir compare="Text">insert-into-loaded-dataset-with-index_02</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-into-loaded-dataset_01">
+ <output-dir compare="Text">insert-into-loaded-dataset_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-into-loaded-dataset_02">
+ <output-dir compare="Text">insert-into-loaded-dataset_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-src-dst-01">
+ <output-dir compare="Text">insert-src-dst-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert">
+ <output-dir compare="Text">insert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-duplicated-keys">
+ <output-dir compare="Text">insert-duplicated-keys</output-dir>
+ <expected-error>Inserting duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-duplicated-keys-with-pk-index">
+ <output-dir compare="Text">insert-duplicated-keys-with-pk-index</output-dir>
+ <expected-error>Inserting duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_less_nc">
+ <output-dir compare="Text">insert_less_nc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="dml">
+ <compilation-unit name="load-from-hdfs">
+ <output-dir compare="Text">load-from-hdfs</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-with-autogenerated-pk_adm-with-sec-primary-index">
+ <output-dir compare="Text">insert-with-autogenerated-pk_adm-with-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-with-autogenerated-pk_adm_01">
+ <output-dir compare="Text">insert-with-autogenerated-pk_adm_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-with-autogenerated-pk_adm_02">
+ <output-dir compare="Text">insert-with-autogenerated-pk_adm_02</output-dir>
+ <expected-error>Field type string cannot be promoted to type uuid</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-with-autogenerated-pk_adm_03">
+ <output-dir compare="Text">insert-with-autogenerated-pk_adm_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-pk_txt_01">
+ <output-dir compare="Text">load-with-autogenerated-pk_txt_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-pk_adm_01">
+ <output-dir compare="Text">load-with-autogenerated-pk_adm_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-pk_adm_02">
+ <output-dir compare="Text">load-with-autogenerated-pk_adm_02</output-dir>
+ <expected-error>ASX3058: This record is closed, you can not add extra fields! new field name: id</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-pk_adm_03">
+ <output-dir compare="Text">load-with-autogenerated-pk_adm_03</output-dir>
+ <expected-error>ASX3058: This record is closed, you can not add extra fields! new field name: id</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-pk_adm_04">
+ <output-dir compare="Text">load-with-autogenerated-pk_adm_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-pk_csv_01">
+ <output-dir compare="Text">load-with-autogenerated-pk_csv_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-pk_csv_02">
+ <output-dir compare="Text">load-with-autogenerated-pk_csv_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-no-field">
+ <output-dir compare="Text">load-with-autogenerated-no-field</output-dir>
+ <expected-error>ASX1014: Field 'not_id' is not found</expected-error>
+ <expected-error>ASX1014: Field 'not_id' is not found</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-return-records">
+ <output-dir compare="Text">insert-return-records</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-returning-udf">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-returning-fieldname">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-returning-fieldname-implicit">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-returning-fieldname-implicit-2">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_uuid_autogenerate">
+ <output-dir compare="Text">insert_uuid_autogenerate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_uuid_autogenerate">
+ <output-dir compare="Text">upsert_uuid_autogenerate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_uuid_manual">
+ <output-dir compare="Text">insert_uuid_manual</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_uuid_manual">
+ <output-dir compare="Text">upsert_uuid_manual</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_uuid_manual_exists">
+ <output-dir compare="Text">insert_uuid_manual_exists</output-dir>
+ <expected-error>Inserting duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_uuid_manual_exists">
+ <output-dir compare="Text">upsert_uuid_manual_exists</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_uuid_manual_exists_select">
+ <output-dir compare="Text">insert_uuid_manual_exists_select</output-dir>
+ <expected-error>Inserting duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_uuid_manual_exists_select">
+ <output-dir compare="Text">upsert_uuid_manual_exists_select</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_nested_uuid_autogenerate">
+ <output-dir compare="Text">insert_nested_uuid_autogenerate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_nested_uuid_autogenerate">
+ <output-dir compare="Text">upsert_nested_uuid_autogenerate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_nested_uuid_manual">
+ <output-dir compare="Text">insert_nested_uuid_manual</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_nested_uuid_manual">
+ <output-dir compare="Text">upsert_nested_uuid_manual</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_nested_uuid_manual_exists">
+ <output-dir compare="Text">insert_nested_uuid_manual_exists</output-dir>
+ <expected-error>Inserting duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_nested_uuid_manual_exists">
+ <output-dir compare="Text">upsert_nested_uuid_manual_exists</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert_nested_uuid_manual_exists_select">
+ <output-dir compare="Text">insert_nested_uuid_manual_exists_select</output-dir>
+ <expected-error>Inserting duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert_nested_uuid_manual_exists_select">
+ <output-dir compare="Text">upsert_nested_uuid_manual_exists_select</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-with-bad-return">
+ <output-dir compare="Text">insert-with-bad-return</output-dir>
+ <expected-error>A returning expression cannot contain dataset access</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-index">
+ <output-dir compare="Text">load-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-ngram-index">
+ <output-dir compare="Text">load-with-ngram-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-rtree-index">
+ <output-dir compare="Text">load-with-rtree-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-word-index">
+ <output-dir compare="Text">load-with-word-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-c2o-recursive">
+ <output-dir compare="Text">opentype-c2o-recursive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-c2o">
+ <output-dir compare="Text">opentype-c2o</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-closed-optional">
+ <output-dir compare="Text">opentype-closed-optional</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-insert">
+ <output-dir compare="Text">opentype-insert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-insert2">
+ <output-dir compare="Text">opentype-insert2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-noexpand">
+ <output-dir compare="Text">opentype-noexpand</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-o2c-recursive">
+ <output-dir compare="Text">opentype-o2c-recursive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-o2c">
+ <output-dir compare="Text">opentype-o2c</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="opentype-o2o">
+ <output-dir compare="Text">opentype-o2o</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit
+ name="scan-delete-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-correlated-secondary-index">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-secondary-index">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-correlated-secondary-index">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-correlated-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-secondary-index-string-as-primary-key">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-correlated-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-nullable-string-as-primary-key">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-nullable-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-correlated-secondary-index-nullable-string-as-primary-key">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-nullable-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-secondary-index-nullable-string-as-primary-key">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-nullable-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-secondary-index-string-as-primary-key">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-correlated-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-correlated-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-index-open">
+ <output-dir compare="Text">load-with-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-ngram-index-open">
+ <output-dir compare="Text">load-with-ngram-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-rtree-index-open">
+ <output-dir compare="Text">load-with-rtree-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-word-index-open">
+ <output-dir compare="Text">load-with-word-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-secondary-index-open">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-secondary-index-open">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index-open">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-secondary-index-open">
+ <output-dir compare="Text">scan-insert-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-open">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-secondary-index-open">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-secondary-index-open">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="delete-multi-statement">
+ <output-dir compare="Text">delete-multi-statement</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert-dataset-with-meta">
+ <output-dir compare="Text">upsert-dataset-with-meta</output-dir>
+ <expected-error>upsert into dataset is not supported on datasets with meta records</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert-return-custom-result">
+ <output-dir compare="Text">upsert-return-custom-result</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert-returning-fieldname">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert-returning-fieldname-implicit">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert-returning-fieldname-implicit-2">
+ <output-dir compare="Text">insert-returning-fieldname</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="delete-dataset-with-meta">
+ <output-dir compare="Text">delete-dataset-with-meta</output-dir>
+ <expected-error>delete from dataset is not supported on datasets with meta records</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-dataset-with-meta">
+ <output-dir compare="Text">insert-dataset-with-meta</output-dir>
+ <expected-error>insert into dataset is not supported on datasets with meta records</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-duplicated-keys-from-query">
+ <output-dir compare="Text">insert-duplicated-keys-from-query</output-dir>
+ <expected-error>HYR0033: Inserting duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-autogenerated-no-field">
+ <output-dir compare="Text">load-with-autogenerated-no-field</output-dir>
+ <expected-error>ASX1014: Field 'not_id' is not found</expected-error>
+ <expected-error>ASX1014: Field 'not_id' is not found</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-with-index-open_02">
+ <output-dir compare="Text">load-with-index-open_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-ASTERIXDB-867">
+ <output-dir compare="Text">query-ASTERIXDB-867</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-ASTERIXDB-1406">
+ <output-dir compare="Text">query-ASTERIXDB-1406</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-issue382">
+ <output-dir compare="Text">query-issue382</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="create-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-into-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert-into-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-into-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="employee">
+ <test-case FilePath="employee">
+ <compilation-unit name="q_01">
+ <output-dir compare="Text">q_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="employee">
+ <compilation-unit name="q_02">
+ <output-dir compare="Text">q_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="failure">
+ <test-case FilePath="failure">
+ <compilation-unit name="group_by_failure">
+ <output-dir compare="Text">group_by_failure</output-dir>
+ <expected-error>Injected failure in inject-failure</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="failure">
+ <compilation-unit name="group_by_hash_failure">
+ <output-dir compare="Text">group_by_hash_failure</output-dir>
+ <expected-error>Injected failure in inject-failure</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="failure">
+ <compilation-unit name="q01_pricing_summary_report_failure">
+ <output-dir compare="Text">q01_pricing_summary_report_failure</output-dir>
+ <expected-error>Injected failure in inject-failure</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="failure">
+ <compilation-unit name="q18_large_volume_customer_failure">
+ <output-dir compare="Text">q18_large_volume_customer_failure</output-dir>
+ <expected-error>Injected failure in inject-failure</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="failure">
+ <compilation-unit name="order_by_failure">
+ <output-dir compare="Text">order_by_failure</output-dir>
+ <expected-error>Injected failure in inject-failure</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="fuzzyjoin">
+ <test-case FilePath="fuzzyjoin">
+ <compilation-unit name="basic-1_1">
+ <output-dir compare="Text">basic-1_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fuzzyjoin">
+ <compilation-unit name="basic-1_1_2">
+ <output-dir compare="Text">basic-1_1_2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fuzzyjoin">
+ <compilation-unit name="basic-1_1_3">
+ <output-dir compare="Text">basic-1_1_3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fuzzyjoin">
+ <compilation-unit name="dblp-string-as-primary-key">
+ <output-dir compare="Text">dblp-string-as-primary-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <!--
+ <test-group name="flwor">
+ <test-case FilePath="flwor">
+ <compilation-unit name="for01">
+ <output-dir compare="Text">for01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for02">
+ <output-dir compare="Text">for02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for03">
+ <output-dir compare="Text">for03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for04">
+ <output-dir compare="Text">for04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for05">
+ <output-dir compare="Text">for05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for06">
+ <output-dir compare="Text">for06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for07">
+ <output-dir compare="Text">for07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for08">
+ <output-dir compare="Text">for08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for09">
+ <output-dir compare="Text">for09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for10">
+ <output-dir compare="Text">for10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for11">
+ <output-dir compare="Text">for11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for12">
+ <output-dir compare="Text">for12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for13">
+ <output-dir compare="Text">for13</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for14">
+ <output-dir compare="Text">for14</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for15">
+ <output-dir compare="Text">for15</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for16">
+ <output-dir compare="Text">for16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for17">
+ <output-dir compare="Text">for17</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for18">
+ <output-dir compare="Text">for18</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="for19">
+ <output-dir compare="Text">for19</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="grpby01">
+ <output-dir compare="Text">grpby01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="grpby02">
+ <output-dir compare="Text">grpby02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let01">
+ <output-dir compare="Text">let01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let02">
+ <output-dir compare="Text">let02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let03">
+ <output-dir compare="Text">let03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let04">
+ <output-dir compare="Text">let04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let05">
+ <output-dir compare="Text">let05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let06">
+ <output-dir compare="Text">let06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let07">
+ <output-dir compare="Text">let07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let08">
+ <output-dir compare="Text">let08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let09">
+ <output-dir compare="Text">let09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let10">
+ <output-dir compare="Text">let10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let11">
+ <output-dir compare="Text">let11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let12">
+ <output-dir compare="Text">let12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let13">
+ <output-dir compare="Text">let13</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let14">
+ <output-dir compare="Text">let14</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let15">
+ <output-dir compare="Text">let15</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let16">
+ <output-dir compare="Text">let16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let17">
+ <output-dir compare="Text">let17</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let18">
+ <output-dir compare="Text">let18</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let19">
+ <output-dir compare="Text">let19</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let20">
+ <output-dir compare="Text">let20</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let21">
+ <output-dir compare="Text">let21</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let22">
+ <output-dir compare="Text">let22</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let23">
+ <output-dir compare="Text">let23</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let24">
+ <output-dir compare="Text">let24</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let25">
+ <output-dir compare="Text">let25</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let26">
+ <output-dir compare="Text">let26</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let27">
+ <output-dir compare="Text">let27</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let28">
+ <output-dir compare="Text">let28</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let29">
+ <output-dir compare="Text">let29</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let30">
+ <output-dir compare="Text">let30</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let31">
+ <output-dir compare="Text">let31</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="let32">
+ <output-dir compare="Text">let32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-01">
+ <output-dir compare="Text">order-by-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-02">
+ <output-dir compare="Text">order-by-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-03">
+ <output-dir compare="Text">order-by-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-04">
+ <output-dir compare="Text">order-by-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-05">
+ <output-dir compare="Text">order-by-05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-06">
+ <output-dir compare="Text">order-by-06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-07">
+ <output-dir compare="Text">order-by-07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-08">
+ <output-dir compare="Text">order-by-08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-09">
+ <output-dir compare="Text">order-by-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-10">
+ <output-dir compare="Text">order-by-10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-11">
+ <output-dir compare="Text">order-by-11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="order-by-12">
+ <output-dir compare="Text">order-by-12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-01">
+ <output-dir compare="Text">ret-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-02">
+ <output-dir compare="Text">ret-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-03">
+ <output-dir compare="Text">ret-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-04">
+ <output-dir compare="Text">ret-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-05">
+ <output-dir compare="Text">ret-05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-06">
+ <output-dir compare="Text">ret-06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-07">
+ <output-dir compare="Text">ret-07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-08">
+ <output-dir compare="Text">ret-08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-09">
+ <output-dir compare="Text">ret-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-10">
+ <output-dir compare="Text">ret-10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-11">
+ <output-dir compare="Text">ret-11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-12">
+ <output-dir compare="Text">ret-12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-13">
+ <output-dir compare="Text">ret-13</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-14">
+ <output-dir compare="Text">ret-14</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-15">
+ <output-dir compare="Text">ret-15</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="writers">
+ <test-case FilePath="writers">
+ <compilation-unit name="print_01">
+ <output-dir compare="Text">print_01</output-dir>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-16">
+ <output-dir compare="Text">ret-16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-17">
+ <output-dir compare="Text">ret-17</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-18">
+ <output-dir compare="Text">ret-18</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
+ <compilation-unit name="ret-19">
+ <output-dir compare="Text">ret-19</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ -->
+ <test-group name="fulltext">
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-01">
+ <output-dir compare="Text">fulltext-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-02">
+ <output-dir compare="Text">fulltext-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-04">
+ <output-dir compare="Text">fulltext-04</output-dir>
+ <expected-error>ASX1010: Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-05">
+ <output-dir compare="Text">fulltext-05</output-dir>
+ <expected-error>ASX1010: Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-06">
+ <output-dir compare="Text">fulltext-06</output-dir>
+ <expected-error>ASX1010: Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-08">
+ <output-dir compare="Text">fulltext-08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-09">
+ <output-dir compare="Text">fulltext-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-01">
+ <output-dir compare="Text">fulltext-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-02">
+ <output-dir compare="Text">fulltext-index-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-04">
+ <output-dir compare="Text">fulltext-index-04</output-dir>
+ <expected-error>ASX1010: Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-05">
+ <output-dir compare="Text">fulltext-index-05</output-dir>
+ <expected-error>ASX1010: Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-06">
+ <output-dir compare="Text">fulltext-index-06</output-dir>
+ <expected-error>ASX1010: Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-08">
+ <output-dir compare="Text">fulltext-index-08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-09">
+ <output-dir compare="Text">fulltext-index-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-large-data">
+ <output-dir compare="Text">fulltext-index-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="stopwords-full-text-filter-1">
+ <output-dir compare="Text">stopwords-full-text-filter-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="global-aggregate">
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q01">
+ <output-dir compare="Text">q01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q02">
+ <output-dir compare="Text">q02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q03">
+ <output-dir compare="Text">q01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q04">
+ <output-dir compare="Text">q01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q05_error">
+ <output-dir compare="Text">q01</output-dir>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier u (in line 22, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q06_error">
+ <output-dir compare="Text">q01</output-dir>
+ <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type string (in line 22, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q07_error">
+ <output-dir compare="Text">q01</output-dir>
+ <expected-error>count is a SQL-92 aggregate function. The SQL++ core aggregate function array_count could potentially express the intent.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q08">
+ <output-dir compare="Text">q08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q09">
+ <output-dir compare="Text">q09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q10">
+ <output-dir compare="Text">q09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q11">
+ <output-dir compare="Text">q01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="q12_error">
+ <output-dir compare="Text">q01</output-dir>
+ <expected-error>The parameter * can only be used in count().</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="query-ASTERIXDB-159">
+ <output-dir compare="Text">query-ASTERIXDB-159</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="query-ASTERIXDB-1626">
+ <output-dir compare="Text">query-ASTERIXDB-1626</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="query-ASTERIXDB-1626-2">
+ <output-dir compare="Text">query-ASTERIXDB-1626-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="global-aggregate">
+ <compilation-unit name="query-ASTERIXDB-2525">
+ <output-dir compare="Text">query-ASTERIXDB-2525</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="group-by">
+ <test-case FilePath="group-by">
+ <compilation-unit name="gby-array">
+ <output-dir compare="Text">gby-array</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="gby-case-01">
+ <output-dir compare="Text">gby-case-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="gby-cross-join">
+ <output-dir compare="Text">gby-cross-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="gby-nested-01">
+ <output-dir compare="Text">gby-nested-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="gby-record">
+ <output-dir compare="Text">gby-record</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-01">
+ <output-dir compare="Text">core-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-01-error">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Cannot find dataset e in dataverse gby nor an alias with name e</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-02-error">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Cannot find dataset f in dataverse gby nor an alias with name f</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-02">
+ <output-dir compare="Text">core-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-03">
+ <output-dir compare="Text">core-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-04">
+ <output-dir compare="Text">core-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-05">
+ <output-dir compare="Text">core-05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-06">
+ <output-dir compare="Text">core-06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="core-07-error">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 25 >>GROUP BY x, y;<< Encountered "GROUP" at column 1]]></expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-01">
+ <output-dir compare="Text">core-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-01-negative">
+ <output-dir compare="Text">core-01</output-dir>
+ <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type bigint (in line 26, at column 26)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-01-2">
+ <output-dir compare="Text">core-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-02">
+ <output-dir compare="Text">core-02</output-dir>
+ <expected-error>Cannot resolve ambiguous alias reference for identifier deptId</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-02-2">
+ <output-dir compare="Text">core-02</output-dir>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier deptId (in line 28, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-03">
+ <output-dir compare="Text">core-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-03-2">
+ <output-dir compare="Text">core-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-04">
+ <output-dir compare="Text">core-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-04-2">
+ <output-dir compare="Text">core-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-05">
+ <output-dir compare="Text">core-05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-05-2">
+ <output-dir compare="Text">core-05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-06-distinct">
+ <output-dir compare="Text">sugar-06-distinct</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-07-negative">
+ <output-dir compare="Text">core-01</output-dir>
+ <expected-error>Cannot resolve ambiguous alias reference for identifier IDENT</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-08-negative">
+ <output-dir compare="Text">core-01</output-dir>
+ <expected-error>ASX1103: Illegal use of identifier: x</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-09">
+ <output-dir compare="Text">sugar-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="two-step-agg-01">
+ <output-dir compare="Text">two-step-agg-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="null">
+ <output-dir compare="Text">null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="gby-expr">
+ <output-dir compare="Text">gby-expr</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-1">
+ <output-dir compare="Text">grouping-sets-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-2">
+ <output-dir compare="Text">grouping-sets-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-3-negative">
+ <output-dir compare="Text">grouping-sets-2</output-dir>
+ <expected-error>ASX1120: Unexpected alias: v21</expected-error>
+ <expected-error>ASX1120: Unexpected alias: v22</expected-error>
+ <expected-error>ASX1120: Unexpected alias: v23</expected-error>
+ <expected-error>ASX1087: Invalid number of arguments for function grouping</expected-error>
+ <expected-error>ASX1119: Invalid argument to grouping() function</expected-error>
+ <expected-error>ASX1119: Invalid argument to grouping() function</expected-error>
+ <expected-error>ASX1119: Invalid argument to grouping() function</expected-error>
+ <expected-error>ASX1118: Too many grouping sets in group by clause: 512. Maximum allowed: 128.</expected-error>
+ <expected-error>ASX1129: Cannot compile SELECT variable.* with GROUP BY GROUPING SETS/ROLLUP/CUBE followed by ORDER BY/LIMIT</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="having">
+ <output-dir compare="Text">core-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="having-2">
+ <output-dir compare="Text">core-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="policy">
+ <output-dir compare="Text">policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="policy-02">
+ <output-dir compare="Text">policy-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="policy-03">
+ <output-dir compare="Text">policy-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="policy-04">
+ <output-dir compare="Text">policy-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by" check-warnings="true">
+ <compilation-unit name="policy-05">
+ <output-dir compare="Text">policy-05</output-dir>
+ <expected-warn>Unsupported type: agg-sum cannot process input type object (in line 29, at column 23)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="listify">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>The byte size of a single group</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="listify-2">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>The byte size of a single group</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="listify-3">
+ <output-dir compare="Text">listify-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="redundant-var-in-groupby">
+ <output-dir compare="Text">redundant-var-in-groupby</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="group-by-all-ASTERIXDB-2611">
+ <output-dir compare="Text">group-by-all-ASTERIXDB-2611</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="hash-group-by-decor">
+ <output-dir compare="Text">hash-group-by-decor</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="group-by">
+ <compilation-unit name="query-ASTERIXDB-3016">
+ <output-dir compare="Text">query-ASTERIXDB-3016</output-dir>
+ </compilation-unit>
+ </test-case-->
+ </test-group>
+ <test-group name="index-join">
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-index-nested-loop-join">
+ <output-dir compare="Text">btree-index-nested-loop-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-primary-equi-join">
+ <output-dir compare="Text">btree-primary-equi-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-pidx-to-sidx-idxonly-equi-join_01">
+ <output-dir compare="Text">btree-pidx-to-sidx-idxonly-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-equi-join_01">
+ <output-dir compare="Text">btree-secondary-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-equi-join_02">
+ <output-dir compare="Text">btree-secondary-equi-join_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-equi-join_03">
+ <output-dir compare="Text">btree-secondary-equi-join_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-equi-join_04">
+ <output-dir compare="Text">btree-secondary-equi-join_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-idxonly-to-pidx-equi-join_01">
+ <output-dir compare="Text">btree-sidx-idxonly-to-pidx-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-idxonly-to-sidx-idxonly-equi-join_01">
+ <output-dir compare="Text">btree-sidx-idxonly-to-sidx-idxonly-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-non-idxonly-to-pidx-equi-join_01">
+ <output-dir compare="Text">btree-sidx-non-idxonly-to-pidx-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-sidx-non-idxonly-to-sidx-idxonly-equi-join_01">
+ <output-dir compare="Text">btree-sidx-non-idxonly-to-sidx-idxonly-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-secondary-self-equi-join_01">
+ <output-dir compare="Text">btree-secondary-self-equi-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join" check-warnings="true">
+ <compilation-unit name="hints-indexnl-params">
+ <output-dir compare="Text">hints-indexnl-params</output-dir>
+ <expected-warn><![CDATA[ASX1132: Invalid specification for hint indexnl. ASX1001: Syntax error: In line 1 >>(8, idx_tenk2_1k_2k)<< Encountered <INTEGER_LITERAL> "8" at column 2. (in line 35, at column 21)]]></expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="rtree-spatial-intersect-point_01">
+ <output-dir compare="Text">rtree-spatial-intersect-point_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="rtree-spatial-intersect-point_02">
+ <output-dir compare="Text">rtree-spatial-intersect-point_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="rtree-spatial-intersect-point_03">
+ <output-dir compare="Text">rtree-spatial-intersect-point_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="rtree-spatial-intersect-point_04">
+ <output-dir compare="Text">rtree-spatial-intersect-point_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="rtree-spatial-intersect-point_05">
+ <output-dir compare="Text">rtree-spatial-intersect-point_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="rtree-spatial-self-intersect-point">
+ <output-dir compare="Text">rtree-spatial-self-intersect-point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-join">
+ <compilation-unit name="btree-multiple-join">
+ <output-dir compare="Text">btree-multiple-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="index-selection">
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-composite-key">
+ <output-dir compare="Text">btree-index-composite-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-composite-key-02">
+ <output-dir compare="Text">btree-index-composite-key-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-composite-key-03">
+ <output-dir compare="Text">btree-index-composite-key-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-composite-key-04">
+ <output-dir compare="Text">btree-index-composite-key-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index">
+ <output-dir compare="Text">btree-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-01">
+ <output-dir compare="Text">btree-sec-primary-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-02">
+ <output-dir compare="Text">btree-sec-primary-index-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-03">
+ <output-dir compare="Text">btree-sec-primary-index-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-04">
+ <output-dir compare="Text">btree-sec-primary-index-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-01">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-02">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-03">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-composite-idxonly-04">
+ <output-dir compare="Text">btree-sidx-composite-idxonly-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-idxonly-01">
+ <output-dir compare="Text">btree-sidx-idxonly-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="index-selection">
+ <compilation-unit name="btree-sidx-non-idxonly-01">
+ <output-dir compare="Text">btree-sidx-non-idxonly-01</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="index-selection">
+ <compilation-unit name="cust-index-age-nullable">
+ <output-dir compare="Text">cust-index-age-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="index-selection">
+ <compilation-unit name="intersection-misc-01">
+ <output-dir compare="Text">intersection-misc-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="intersection-with-between">
+ <output-dir compare="Text">intersection-with-between</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-contains">
+ <output-dir compare="Text">inverted-index-ngram-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-panic">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-panic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-large-data">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-word-tokens">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-word-tokens</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-jaccard">
+ <output-dir compare="Text">inverted-index-ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-contains">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-olist-edit-distance-panic">
+ <output-dir compare="Text">inverted-index-olist-edit-distance-panic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-olist-edit-distance">
+ <output-dir compare="Text">inverted-index-olist-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-olist-jaccard">
+ <output-dir compare="Text">inverted-index-olist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ulist-jaccard">
+ <output-dir compare="Text">inverted-index-ulist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-word-contains">
+ <output-dir compare="Text">inverted-index-word-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-word-jaccard">
+ <output-dir compare="Text">inverted-index-word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive-open">
+ <output-dir compare="Text">orders-index-custkey-conjunctive-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey-open">
+ <output-dir compare="Text">orders-index-custkey-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="range-search-open">
+ <output-dir compare="Text">range-search-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="rtree-secondary-index-nullable">
+ <output-dir compare="Text">rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="rtree-secondary-index-open">
+ <output-dir compare="Text">rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="rtree-secondary-index">
+ <output-dir compare="Text">rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="rtree-secondary-index-circular-query">
+ <output-dir compare="Text">rtree-secondary-index-circular-query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="rtree-sidx-idxonly-01">
+ <output-dir compare="Text">rtree-sidx-idxonly-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="index-selection">
+ <compilation-unit name="rtree-sidx-non-idxonly-01">
+ <output-dir compare="Text">rtree-sidx-non-idxonly-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="rtree-sidx-non-idxonly-02">
+ <output-dir compare="Text">rtree-sidx-non-idxonly-02</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="index-selection">
+ <compilation-unit name="disjunctive-predicate-1">
+ <output-dir compare="Text">disjunctive-predicate-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="dataset-with-meta">
+ <output-dir compare="Text">dataset-with-meta</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="index-selection" check-warnings="true">
+ <compilation-unit name="hints-skip-index">
+ <output-dir compare="Text">hints-skip-index</output-dir>
+ <expected-warn><![CDATA[ASX1132: Invalid specification for hint skip-index. ASX1001: Syntax error: In line 1 >>(13, idx_1k)<< Encountered <INTEGER_LITERAL> "13" at column 2. (in line 32, at column 19)]]></expected-warn>
+ </compilation-unit>
+ </test-case!-->
+ <!--test-case FilePath="index-selection" check-warnings="true">
+ <compilation-unit name="hints-use-index">
+ <output-dir compare="Text">hints-use-index</output-dir>
+ <expected-warn><![CDATA[ASX1132: Invalid specification for hint use-index. ASX1001: Syntax error: In line 1 >>(18, idx_1k_2k)<< Encountered <INTEGER_LITERAL> "18" at column 2. (in line 33, at column 15)]]></expected-warn>
+ <expected-warn><![CDATA[ASX1132: Invalid specification for hint use-index. ASX1001: Syntax error: In line 1 >>()<< Encountered ")" at column 2. (in line 33, at column 15)]]></expected-warn>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-with-two-ngram-index">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-with-two-ngram-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="index-selection">
+ <compilation-unit name="intersection">
+ <output-dir compare="Text">intersection</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="intersection-with-filter">
+ <output-dir compare="Text">intersection-with-filter</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="intersection_with_nodegroup">
+ <output-dir compare="Text">intersection</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="index-selection">
+ <compilation-unit name="verify">
+ <output-dir compare="Text">verify</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="inverted-index-join">
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="ngram-edit-distance">
+ <output-dir compare="Text">ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="ngram-edit-distance-inline">
+ <output-dir compare="Text">ngram-edit-distance-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="ngram-jaccard">
+ <output-dir compare="Text">ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="ngram-jaccard-inline">
+ <output-dir compare="Text">ngram-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="olist-edit-distance">
+ <output-dir compare="Text">olist-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="olist-edit-distance-inline">
+ <output-dir compare="Text">olist-edit-distance-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="olist-jaccard">
+ <output-dir compare="Text">olist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="olist-jaccard-inline">
+ <output-dir compare="Text">olist-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="ulist-jaccard">
+ <output-dir compare="Text">ulist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="ulist-jaccard-inline">
+ <output-dir compare="Text">ulist-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="word-jaccard">
+ <output-dir compare="Text">word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join">
+ <compilation-unit name="word-jaccard-inline">
+ <output-dir compare="Text">word-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="cast-default-null">
+ <placeholder name="with" value="" />
+ <output-dir compare="Text">cast-default-null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="cast-default-null">
+ <placeholder name="with" value="WITH {'merge-policy': {'name': 'correlated-prefix','parameters': { 'max-mergable-component-size': 16384, 'max-tolerance-component-count': 3 }}}" />
+ <output-dir compare="Text">cast-default-null</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="inverted-index-join-noeqjoin">
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="ngram-edit-distance">
+ <output-dir compare="Text">ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="ngram-edit-distance-inline">
+ <output-dir compare="Text">ngram-edit-distance-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="ngram-jaccard">
+ <output-dir compare="Text">ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="ngram-jaccard-inline">
+ <output-dir compare="Text">ngram-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="olist-edit-distance">
+ <output-dir compare="Text">olist-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="olist-edit-distance-inline">
+ <output-dir compare="Text">olist-edit-distance-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="olist-jaccard">
+ <output-dir compare="Text">olist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="olist-jaccard-inline">
+ <output-dir compare="Text">olist-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="ulist-jaccard">
+ <output-dir compare="Text">ulist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="ulist-jaccard-inline">
+ <output-dir compare="Text">ulist-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="word-jaccard">
+ <output-dir compare="Text">word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="inverted-index-join-noeqjoin">
+ <compilation-unit name="word-jaccard-inline">
+ <output-dir compare="Text">word-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="join">
+ <test-case FilePath="join">
+ <compilation-unit name="cross-join-01">
+ <output-dir compare="Text">cross-join-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="join">
+ <compilation-unit name="cross-join-02-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1077: Cannot find dataset x in dataverse Default nor an alias with name x (in line 26, at column 39)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="join">
+ <compilation-unit name="hash_join_array">
+ <output-dir compare="Text">hash_join_array</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="join">
+ <compilation-unit name="hash_join_missing">
+ <output-dir compare="Text">hash_join_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="join">
+ <compilation-unit name="hash_join_record">
+ <output-dir compare="Text">hash_join_record</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="join">
+ <compilation-unit name="join-with-empty-dataset">
+ <output-dir compare="Text">join-with-empty-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="list">
+ <test-case FilePath="list">
+ <compilation-unit name="any-collection-member_01">
+ <output-dir compare="Text">any-collection-member_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="array_length">
+ <output-dir compare="Text">array_length</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="enforcing_item_type">
+ <output-dir compare="Text">enforcing_item_type</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="exists">
+ <output-dir compare="Text">exists</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="get-item_01">
+ <output-dir compare="Text">get-item_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="get-item_02">
+ <output-dir compare="Text">get-item_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="get-item_03">
+ <output-dir compare="Text">get-item_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="len_01">
+ <output-dir compare="Text">len_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="len_null_01">
+ <output-dir compare="Text">len_null_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="list-range">
+ <output-dir compare="Text">list-range</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="list-slice_01">
+ <output-dir compare="Text">list-slice_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="list-slice_02">
+ <output-dir compare="Text">list-slice_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="list-slice_03">
+ <output-dir compare="Text">list-slice_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="list-star_01">
+ <output-dir compare="Text">list-star_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="listify_01">
+ <output-dir compare="Text">listify_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="listify_02">
+ <output-dir compare="Text">listify_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="listify_03">
+ <output-dir compare="Text">listify_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="ordered-list-constructor_01">
+ <output-dir compare="Text">ordered-list-constructor_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="ordered-list-constructor_02">
+ <output-dir compare="Text">ordered-list-constructor_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="ordered-list-constructor_03">
+ <output-dir compare="Text">ordered-list-constructor_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="ordered-list-constructor_04">
+ <output-dir compare="Text">ordered-list-constructor_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="ordered-list-constructor_05">
+ <output-dir compare="Text">ordered-list-constructor_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="ordered-list-constructor_06_negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 26 >> from [r, r+1] x select value count(x),<< Encountered "," at column 42]]></expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="scan-collection_01">
+ <output-dir compare="Text">scan-collection_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- <test-case FilePath="list">
+ <compilation-unit name="union_01">
+ <output-dir compare="Text">union_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="union_02">
+ <output-dir compare="Text">union_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="list">
+ <compilation-unit name="unordered-list-constructor_01">
+ <output-dir compare="Text">unordered-list-constructor_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="unordered-list-constructor_02">
+ <output-dir compare="Text">unordered-list-constructor_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="unordered-list-constructor_03">
+ <output-dir compare="Text">unordered-list-constructor_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-issue428">
+ <output-dir compare="Text">query-issue428</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-159-2">
+ <output-dir compare="Text">query-ASTERIXDB-159-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-159-3">
+ <output-dir compare="Text">query-ASTERIXDB-159-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-1131-2">
+ <output-dir compare="Text">query-ASTERIXDB-1131-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-1131">
+ <output-dir compare="Text">query-ASTERIXDB-1131</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-1212-2-open">
+ <output-dir compare="Text">query-ASTERIXDB-1212-2-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-1212-2">
+ <output-dir compare="Text">query-ASTERIXDB-1212-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-1212-open">
+ <output-dir compare="Text">query-ASTERIXDB-1212-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-1212">
+ <output-dir compare="Text">query-ASTERIXDB-1212</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-ASTERIXDB-673">
+ <output-dir compare="Text">query-ASTERIXDB-673</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="var-in-list">
+ <output-dir compare="Text">var-in-list</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="misc">
+ <test-case FilePath="misc">
+ <compilation-unit name="big_in_list/000">
+ <output-dir compare="Text">big_in_list/000</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="big_in_list/001">
+ <output-dir compare="Text">big_in_list/001</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="multiple_and/000">
+ <output-dir compare="Text">multiple_and/000</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="record-serialization-ASTERIXDB-2567">
+ <output-dir compare="Text">record-serialization-ASTERIXDB-2567</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="record-serialization-ASTERIXDB-2613">
+ <output-dir compare="Text">record-serialization-ASTERIXDB-2613</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="field_access-ASTERIXDB-2289">
+ <output-dir compare="Text">field_access-ASTERIXDB-2289</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="comp-ASTERIXDB-2415">
+ <output-dir compare="Text">query-ASTERIXDB-1671</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="comp-ASTERIXDB-2412">
+ <output-dir compare="Text">comp-ASTERIXDB-2412</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="field_access_union-ASTERIXDB-2288">
+ <output-dir compare="Text">field_access_union-ASTERIXDB-2288</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="constant_folding">
+ <output-dir compare="Text">constant_folding</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="join-ASTERIXDB-2686">
+ <output-dir compare="Text">join-ASTERIXDB-2686</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="poll-dynamic">
+ <output-dir compare="Text">poll-dynamic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="validate-expected">
+ <output-dir compare="Text">validate-expected</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="dataset-resources">
+ <output-dir compare="Text">dataset-resources</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="ping">
+ <output-dir compare="Text">ping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_01">
+ <output-dir compare="Text">case_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_02">
+ <output-dir compare="Text">case_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_03">
+ <output-dir compare="Text">case_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_04">
+ <output-dir compare="Text">case_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_05">
+ <output-dir compare="Text">case_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_06">
+ <output-dir compare="Text">case_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_07">
+ <output-dir compare="Text">case_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_08">
+ <output-dir compare="Text">case_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="case_09">
+ <output-dir compare="Text">case_09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="dataset_nodegroup">
+ <output-dir compare="Text">dataset_nodegroup</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="ensure_result_numeric_type">
+ <output-dir compare="Text">ensure_result_numeric_type</output-dir>
+ <expected-error>expected < 3.0</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="partition-by-nonexistent-field">
+ <output-dir compare="Text">partition-by-nonexistent-field</output-dir>
+ <expected-error>Field 'id' is not found</expected-error>
+ <expected-error>Cannot find dataset with name testds in dataverse test</expected-error>
+ <expected-error>Cannot find dataset testds in dataverse test nor an alias with name testds</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="float_01">
+ <output-dir compare="Text">float_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="flushtest">
+ <output-dir compare="Text">flushtest</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="groupby-orderby-count">
+ <output-dir compare="Text">groupby-orderby-count</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="identifier_01">
+ <output-dir compare="Text">identifier_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="ifthenelse_01">
+ <output-dir compare="Text">ifthenelse_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="is-null_01">
+ <output-dir compare="Text">is-null_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="nested-loop-join_01">
+ <output-dir compare="Text">nested-loop-join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query_issue267">
+ <output-dir compare="Text">query_issue267</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="random">
+ <output-dir compare="Text">random</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="stable_sort">
+ <output-dir compare="Text">stable_sort</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="misc">
+ <compilation-unit name="range_01">
+ <output-dir compare="Text">range_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="misc">
+ <compilation-unit name="tid_01">
+ <output-dir compare="Text">tid_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="misc">
+ <compilation-unit name="year_01">
+ <output-dir compare="Text">year_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="string_eq_01">
+ <output-dir compare="Text">string_eq_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="prefix-search">
+ <output-dir compare="Text">prefix-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-1490">
+ <output-dir compare="Text">query-ASTERIXDB-1490</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-971">
+ <output-dir compare="Text">query-ASTERIXDB-971-sqlpp</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-1531">
+ <output-dir compare="Text">query-ASTERIXDB-1531</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-1577">
+ <output-dir compare="Text">query-ASTERIXDB-1577</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-1671">
+ <output-dir compare="Text">query-ASTERIXDB-1671</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-1671-2">
+ <output-dir compare="Text">query-ASTERIXDB-1671</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2354">
+ <output-dir compare="Text">query-ASTERIXDB-2354</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2355">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 22 >> %%%<< Encountered "%" at column 2.]]></expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2380">
+ <output-dir compare="Text">query-ASTERIXDB-2380</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2550">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type object (in line 28, at column 2)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2886">
+ <output-dir compare="Text">query-ASTERIXDB-2886</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="unsupported_parameter">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Query parameter compiler.joinmem is not supported</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="unsupported_parameter_value">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>The given string: 4LS is not a byte unit string (e.g., 320KB or 1024)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="uuid">
+ <output-dir compare="Text">uuid</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="p_sort_join">
+ <output-dir compare="Text">p_sort_join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="p_sort_seq_merge">
+ <output-dir compare="Text">p_sort_seq_merge</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="p_sort_num_samples">
+ <output-dir compare="Text">p_sort_num_samples</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="p_sort_static_range_map">
+ <output-dir compare="Text">p_sort_static_range_map</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="active_requests">
+ <output-dir compare="Text">active_requests</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="jobs">
+ <output-dir compare="Text">jobs</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="completed_requests">
+ <output-dir compare="Text">completed_requests</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="dump_index">
+ <output-dir compare="Text">dump_index</output-dir>
+ <expected-error>Cannot find index with name noindex</expected-error>
+ <expected-error>Cannot find dataset with name nodataset in dataverse test</expected-error>
+ <expected-error>Cannot find dataset with name ds in dataverse nodataverse</expected-error>
+ <expected-error>Unsupported type: dump-index cannot process input type null</expected-error>
+ <expected-error>Unsupported type: dump-index cannot process input type null</expected-error>
+ <expected-error>Unsupported type: dump-index cannot process input type null</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="metadata_only_01">
+ <output-dir compare="Text">metadata_only_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="cast-ASTERIXDB-2458">
+ <output-dir compare="Text">cast-ASTERIXDB-2458</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="insert_nulls_with_secondary_idx">
+ <output-dir compare="Text">insert_nulls_with_secondary_idx</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2700">
+ <output-dir compare="Text">query-ASTERIXDB-2700</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-1203">
+ <output-dir compare="Text">query-ASTERIXDB-1203</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-819-2">
+ <output-dir compare="Text">query-ASTERIXDB-819-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-819">
+ <output-dir compare="Text">query-ASTERIXDB-819</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-865">
+ <output-dir compare="Text">query-ASTERIXDB-865</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="multipart-dataverse">
+ <test-case FilePath="multipart-dataverse">
+ <compilation-unit name="index_1">
+ <output-dir compare="Text">index_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="multipart-dataverse">
+ <compilation-unit name="resolution_1">
+ <output-dir compare="Text">resolution_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="multipart-dataverse">
+ <compilation-unit name="special_chars_1">
+ <output-dir compare="Text">special_chars_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="multipart-dataverse">
+ <compilation-unit name="special_chars_2">
+ <output-dir compare="Text">special_chars_2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="multipart-dataverse">
+ <compilation-unit name="udf_1">
+ <output-dir compare="Text">udf_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="index">
+ <test-group name="index/validations">
+ <test-case FilePath="index/validations">
+ <compilation-unit name="keys-same-as-pk-but-different-order">
+ <output-dir compare="Text">keys-same-as-pk-but-different-order</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index/validations">
+ <compilation-unit name="keys-same-as-pk-in-same-order">
+ <output-dir compare="Text">keys-same-as-pk-in-same-order</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index/validations">
+ <compilation-unit name="repetitive-keys">
+ <output-dir compare="Text">repetitive-keys</output-dir>
+ <expected-error>Cannot create index with the same field '[value]' specified more than once.</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ </test-group>
+ <test-group name="open-index-enforced">
+ <test-group name="open-index-enforced/error-checking">
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="enforced-field-name-collision">
+ <output-dir compare="Text">enforced-field-name-collision</output-dir>
+ <expected-error>Cannot create enforced index on '[value]' field. The field is closed type.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="enforced-field-type-collision">
+ <output-dir compare="Text">enforced-field-type-collision</output-dir>
+ <expected-error>Cannot create enforced index on '[value]' field. The field is closed type.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="missing-enforce-statement">
+ <output-dir compare="Text">missing-enforce-statement</output-dir>
+ <expected-error>ASX1042: Cannot create non-enforced typed index of this kind: RTREE</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="missing-optionality">
+ <output-dir compare="Text">missing-optionality</output-dir>
+ <expected-error>Cannot create enforced index on '[value]' field with non-optional type</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="index-on-closed-type">
+ <output-dir compare="Text">index-on-closed-type</output-dir>
+ <expected-error>ASX1014: Field 'value' is not found (in line 33, at column 34)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="index-type-collision">
+ <output-dir compare="Text">index-type-collision</output-dir>
+ <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field(s) 'value' is already defined with type(s) 'integer'</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="index-type-promotion-collision">
+ <output-dir compare="Text">index-type-promotion-collision</output-dir>
+ <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field(s) 'value' is already defined with type(s) 'bigint'</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/error-checking">
+ <compilation-unit name="object-type-collision">
+ <output-dir compare="Text">object-type-collision</output-dir>
+ <expected-error>ASX1051: Cannot create enforced index on '[value]' field. The field is closed type.</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="open-index-enforced/index-join">
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="btree-secondary-equi-join">
+ <output-dir compare="Text">btree-secondary-equi-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="ngram-edit-distance">
+ <output-dir compare="Text">ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="ngram-edit-distance-inline">
+ <output-dir compare="Text">ngram-edit-distance-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="ngram-jaccard">
+ <output-dir compare="Text">ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="ngram-jaccard-inline">
+ <output-dir compare="Text">ngram-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="rtree-spatial-intersect-point">
+ <output-dir compare="Text">rtree-spatial-intersect-point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="word-jaccard">
+ <output-dir compare="Text">word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-join">
+ <compilation-unit name="word-jaccard-inline">
+ <output-dir compare="Text">word-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="open-index-enforced/index-leftouterjoin">
+ <test-case FilePath="open-index-enforced/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="open-index-enforced/index-selection">
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="btree-index-composite-key">
+ <output-dir compare="Text">btree-index-composite-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-ngram-contains">
+ <output-dir compare="Text">inverted-index-ngram-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-contains">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-panic">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-panic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-word-tokens">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-word-tokens</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-ngram-jaccard">
+ <output-dir compare="Text">inverted-index-ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-word-contains">
+ <output-dir compare="Text">inverted-index-word-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="inverted-index-word-jaccard">
+ <output-dir compare="Text">inverted-index-word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="rtree-secondary-index">
+ <output-dir compare="Text">rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="multi-index-composite-key">
+ <output-dir compare="Text">multi-index-composite-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/index-selection">
+ <compilation-unit name="multi-index">
+ <output-dir compare="Text">multi-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="open-index-enforced/type-checking">
+ <test-case FilePath="open-index-enforced/type-checking">
+ <compilation-unit name="enforced-type-delete">
+ <output-dir compare="Text">enforced-type-delete</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-enforced/type-checking">
+ <compilation-unit name="enforced-type-upsert">
+ <output-dir compare="Text">enforced-type-upsert</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="open-index-non-enforced/index-selection">
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-01">
+ <output-dir compare="Text">btree-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-02">
+ <output-dir compare="Text">btree-index-02</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-03">
+ <output-dir compare="Text">btree-index-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-04">
+ <output-dir compare="Text">btree-index-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-composite-key-03">
+ <output-dir compare="Text">btree-index-composite-key-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/index-selection">
+ <compilation-unit name="btree-index-composite-key-04">
+ <output-dir compare="Text">btree-index-composite-key-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-index-non-enforced/correlated-index-selection">
+ <compilation-unit name="btree-index-01">
+ <output-dir compare="Text">btree-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="open-index-non-enforced/index-join">
+ <test-case FilePath="open-index-non-enforced/index-join">
+ <compilation-unit name="btree-equi-join-01">
+ <output-dir compare="Text">btree-equi-join-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ </test-group>
+ <test-group name="nested-open-index">
+ <test-group name="nested-open-index/index-join">
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="btree-secondary-equi-join">
+ <output-dir compare="Text">btree-secondary-equi-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="ngram-edit-distance">
+ <output-dir compare="Text">ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="ngram-edit-distance-inline">
+ <output-dir compare="Text">ngram-edit-distance-inline</output-dir>
+ </compilation-unit>
+ </test-case> -->
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="ngram-jaccard">
+ <output-dir compare="Text">ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="ngram-jaccard-inline">
+ <output-dir compare="Text">ngram-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="rtree-spatial-intersect-point">
+ <output-dir compare="Text">rtree-spatial-intersect-point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="word-jaccard">
+ <output-dir compare="Text">word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="word-jaccard-inline">
+ <output-dir compare="Text">word-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="btree-secondary-non-enforced-equi-join">
+ <output-dir compare="Text">btree-secondary-non-enforced-equi-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-join">
+ <compilation-unit name="btree-secondary-non-enforced-equi-join-2">
+ <output-dir compare="Text">btree-secondary-non-enforced-equi-join-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="nested-open-index/index-leftouterjoin">
+ <test-case FilePath="nested-open-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="nested-open-index/index-selection">
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="btree-index-composite-key">
+ <output-dir compare="Text">btree-index-composite-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-contains">
+ <output-dir compare="Text">inverted-index-ngram-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-contains">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-panic">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-panic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-word-tokens">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-word-tokens</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-jaccard">
+ <output-dir compare="Text">inverted-index-ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-word-contains">
+ <output-dir compare="Text">inverted-index-word-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="inverted-index-word-jaccard">
+ <output-dir compare="Text">inverted-index-word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="rtree-secondary-index">
+ <output-dir compare="Text">rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="non-enforced-01">
+ <output-dir compare="Text">non-enforced-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="non-enforced-02">
+ <output-dir compare="Text">non-enforced-02</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="non-enforced-03">
+ <output-dir compare="Text">non-enforced-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/index-selection">
+ <compilation-unit name="non-enforced-04">
+ <output-dir compare="Text">non-enforced-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="nested-open-index/highly-open-highly-nested">
+ <test-case FilePath="nested-open-index/highly-open-highly-nested">
+ <compilation-unit name="bottom-closed-top-closed">
+ <output-dir compare="Text">bottom-closed-top-closed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/highly-open-highly-nested">
+ <compilation-unit name="bottom-closed-top-open">
+ <output-dir compare="Text">bottom-closed-top-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/highly-open-highly-nested">
+ <compilation-unit name="bottom-open-top-closed">
+ <output-dir compare="Text">bottom-open-top-closed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-open-index/highly-open-highly-nested">
+ <compilation-unit name="bottom-open-top-open">
+ <output-dir compare="Text">bottom-open-top-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ </test-group>
+ <test-group name="nested-index">
+ <test-group name="nested-index/index-join">
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="btree-primary-equi-join">
+ <output-dir compare="Text">btree-primary-equi-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="btree-secondary-equi-join">
+ <output-dir compare="Text">btree-secondary-equi-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="ngram-edit-distance">
+ <output-dir compare="Text">ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="ngram-edit-distance-inline">
+ <output-dir compare="Text">ngram-edit-distance-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="ngram-jaccard">
+ <output-dir compare="Text">ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="ngram-jaccard-inline">
+ <output-dir compare="Text">ngram-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="rtree-spatial-intersect-point">
+ <output-dir compare="Text">rtree-spatial-intersect-point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="word-jaccard">
+ <output-dir compare="Text">word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-join">
+ <compilation-unit name="word-jaccard-inline">
+ <output-dir compare="Text">word-jaccard-inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="nested-index/index-leftouterjoin">
+ <test-case FilePath="nested-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="nested-index/index-selection">
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="btree-index-composite-key">
+ <output-dir compare="Text">btree-index-composite-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="btree-sec-primary-index">
+ <output-dir compare="Text">btree-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="btree-index-composite-key-mixed-intervals">
+ <output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="btree-index-rewrite-multiple">
+ <output-dir compare="Text">btree-index-rewrite-multiple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="cust-index-age-nullable">
+ <output-dir compare="Text">cust-index-age-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-contains">
+ <output-dir compare="Text">inverted-index-ngram-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-contains">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-panic">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-panic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-word-tokens">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-word-tokens</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ngram-jaccard">
+ <output-dir compare="Text">inverted-index-ngram-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-olist-edit-distance">
+ <output-dir compare="Text">inverted-index-olist-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-olist-edit-distance-panic">
+ <output-dir compare="Text">inverted-index-olist-edit-distance-panic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-olist-jaccard">
+ <output-dir compare="Text">inverted-index-olist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-ulist-jaccard">
+ <output-dir compare="Text">inverted-index-ulist-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-word-contains">
+ <output-dir compare="Text">inverted-index-word-contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="inverted-index-word-jaccard">
+ <output-dir compare="Text">inverted-index-word-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey">
+ <output-dir compare="Text">orders-index-custkey</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive">
+ <output-dir compare="Text">orders-index-custkey-conjunctive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey-conjunctive-open">
+ <output-dir compare="Text">orders-index-custkey-conjunctive-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="orders-index-custkey-open">
+ <output-dir compare="Text">orders-index-custkey-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="range-search">
+ <output-dir compare="Text">range-search</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="range-search-open">
+ <output-dir compare="Text">range-search-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="rtree-secondary-index">
+ <output-dir compare="Text">rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="rtree-secondary-index-nullable">
+ <output-dir compare="Text">rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="rtree-secondary-index-open">
+ <output-dir compare="Text">rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index/index-selection">
+ <compilation-unit name="rtree-secondary-index-optional">
+ <output-dir compare="Text">rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ </test-group>
+ <test-group name="nested-index-dml">
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="compact-dataset-and-its-indexes">
+ <output-dir compare="Text">compact-dataset-and-its-indexes</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="nested-uuid-load">
+ <output-dir compare="Text">nested-uuid-load</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="nested-uuid-insert">
+ <output-dir compare="Text">nested-uuid-insert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="delete-from-loaded-dataset-with-index">
+ <output-dir compare="Text">delete-from-loaded-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="delete-from-loaded-dataset-with-sec-primary-index">
+ <output-dir compare="Text">delete-from-loaded-dataset-with-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="drop-index">
+ <output-dir compare="Text">drop-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="nested-index-dml">
+ <compilation-unit name="insert-into-empty-dataset-with-index">
+ <output-dir compare="Text">insert-into-empty-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="insert-into-empty-dataset-with-sec-primary-index">
+ <output-dir compare="Text">insert-into-empty-dataset-with-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="nested-index-dml">
+ <compilation-unit name="insert-into-loaded-dataset-with-index_01">
+ <output-dir compare="Text">insert-into-loaded-dataset-with-index_01</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <!--test-case FilePath="nested-index-dml">
+ <compilation-unit name="insert-into-loaded-dataset-with-index_02">
+ <output-dir compare="Text">insert-into-loaded-dataset-with-index_02</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="insert-into-loaded-dataset-with-sec-primary-index">
+ <output-dir compare="Text">insert-into-loaded-dataset-with-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-index">
+ <output-dir compare="Text">load-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-sec-primary-index">
+ <output-dir compare="Text">load-with-sec-primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-ngram-index">
+ <output-dir compare="Text">load-with-ngram-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-rtree-index">
+ <output-dir compare="Text">load-with-rtree-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="load-with-word-index">
+ <output-dir compare="Text">load-with-word-index</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-btree-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-rtree-secondary-index">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-insert-btree-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-insert-rtree-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-insert-rtree-secondary-index">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-insert-inverted-index-word-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-insert-inverted-index-word-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-inverted-index-word-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nested-index-dml">
+ <compilation-unit name="scan-delete-inverted-index-word-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index">
+ <test-group name="array-index/error-handling">
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-include-unknown-key">
+ <output-dir compare="Text">index-include-unknown-key</output-dir>
+ <expected-error>ASX1079: Compilation error: Array indexes must specify EXCLUDE UNKNOWN KEY</expected-error>
+ <expected-error>ASX1079: Compilation error: Array indexes must specify EXCLUDE UNKNOWN KEY</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-two-array-fields">
+ <output-dir compare="Text">index-two-array-fields</output-dir>
+ <expected-error>ASX1079: Compilation error: Cannot create composite index with multiple array fields using different arrays</expected-error>
+ <expected-error>ASX1079: Compilation error: Cannot create composite index with multiple array fields using different arrays</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="invalid-array-path">
+ <output-dir compare="Text">invalid-array-path</output-dir>
+ <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type CheckinType_checkin_time:</expected-error>
+ <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type string</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-on-closed-array">
+ <output-dir compare="Text">index-on-closed-array</output-dir>
+ <expected-error>ASX1014: Field 'date' is not found</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-with-enforced-type">
+ <output-dir compare="Text">index-with-enforced-type</output-dir>
+ <expected-error>ASX1155: Incompatible index type ARRAY</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/metadata">
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="complex-structures">
+ <output-dir compare="Text">complex-structures</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="with-composite-array-different-indicators">
+ <output-dir compare="Text">with-composite-array-different-indicators</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="with-composite-array-different-indicators">
+ <output-dir compare="Text">with-composite-array-different-indicators</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/bulk-loading/on-index-creation">
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="composite-atomic">
+ <output-dir compare="Text">composite-atomic</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/bulk-loading/after-index-creation">
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-open-index">
+ <output-dir compare="Text">with-open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/insert-upsert-delete">
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="with-additional-atomic-index">
+ <output-dir compare="Text">with-additional-atomic-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="with-additional-atomic-index">
+ <output-dir compare="Text">with-additional-atomic-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/join-quantified-queries">
+ <test-case FilePath="array-index/join-quantified-queries">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-quantified-queries">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-quantified-queries">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-quantified-queries">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/join-unnest-queries">
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="with-open-index">
+ <output-dir compare="Text">with-open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="loj-subquery">
+ <output-dir compare="Text">loj-subquery</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/select-unnest-queries">
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="using-feed-new-index">
+ <output-dir compare="Text">using-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="using-feed-old-index">
+ <output-dir compare="Text">using-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/select-quantified-queries">
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="with-open-index">
+ <output-dir compare="Text">with-open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/composite-index-queries">
+ <test-case FilePath="array-index">
+ <compilation-unit name="composite-index-queries">
+ <output-dir compare="Text">composite-index-queries</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ </test-group>
+ <test-group name="nestrecords">
+ <test-case FilePath="nestrecords">
+ <compilation-unit name="nestrecord">
+ <output-dir compare="Text">nestrecord</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nestrecords">
+ <compilation-unit name="nested-optional-pk">
+ <output-dir compare="Text">nested-optional-pk</output-dir>
+ <expected-error>ASX1021: The primary key field 'nested.id' cannot be nullable</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nestrecords">
+ <compilation-unit name="query-ASTERIXDB-1025">
+ <output-dir compare="Text">query-ASTERIXDB-1025</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="null-missing">
+ <test-case FilePath="null-missing">
+ <compilation-unit name="array">
+ <output-dir compare="Text">array</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="boolean">
+ <output-dir compare="Text">boolean</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="ceil">
+ <output-dir compare="Text">ceil</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="comparison">
+ <output-dir compare="Text">comparison</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="field-access">
+ <output-dir compare="Text">field-access</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="group-by">
+ <output-dir compare="Text">group-by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="group-by-from-dataset">
+ <output-dir compare="Text">group-by-from-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="length">
+ <output-dir compare="Text">length</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="lower">
+ <output-dir compare="Text">lower</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="order-by">
+ <output-dir compare="Text">order-by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="order-by-2">
+ <output-dir compare="Text">order-by-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="order-by-3-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 33 >>ORDER BY d.a NULLS;<< Encountered ";" at column 19]]></expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: Unexpected token: NULLS (in line 33, at column 20)]]></expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="order-by-from-dataset">
+ <output-dir compare="Text">order-by-from-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="order-by-from-dataset-2">
+ <output-dir compare="Text">order-by-from-dataset-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="scan-collection">
+ <output-dir compare="Text">scan-collection</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="spatial-intersect">
+ <output-dir compare="Text">spatial-intersect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="substr">
+ <output-dir compare="Text">substr</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="upper">
+ <output-dir compare="Text">upper</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="is">
+ <output-dir compare="Text">is</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="ifmissing">
+ <output-dir compare="Text">ifmissing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="ifnull">
+ <output-dir compare="Text">ifnull</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="ifmissingornull">
+ <output-dir compare="Text">ifmissingornull</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="coalesce">
+ <output-dir compare="Text">coalesce</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="query-ASTERIXDB-1689">
+ <output-dir compare="Text">query-ASTERIXDB-1689</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="null-missing">
+ <compilation-unit name="print-ASTERIXDB-1885">
+ <output-dir compare="Text">print-ASTERIXDB-1885</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="numeric">
+ <test-case FilePath="numeric">
+ <compilation-unit name="caret0">
+ <output-dir compare="Text">caret0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="caret1">
+ <output-dir compare="Text">caret1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="abs0">
+ <output-dir compare="Text">abs0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="abs1">
+ <output-dir compare="Text">abs1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="abs2">
+ <output-dir compare="Text">abs2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="abs3">
+ <output-dir compare="Text">abs3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="abs4">
+ <output-dir compare="Text">abs4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="add_double">
+ <output-dir compare="Text">add_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="add_float">
+ <output-dir compare="Text">add_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="add_int16">
+ <output-dir compare="Text">add_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="add_int32">
+ <output-dir compare="Text">add_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="add_int64">
+ <output-dir compare="Text">add_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="add_int8">
+ <output-dir compare="Text">add_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="numeric">
+ <compilation-unit name="issue_1166">
+ <output-dir compare="Text">issue_1166</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="numeric">
+ <compilation-unit name="ceiling0">
+ <output-dir compare="Text">ceiling0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ceiling1">
+ <output-dir compare="Text">ceiling1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ceiling2">
+ <output-dir compare="Text">ceiling2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ceiling3">
+ <output-dir compare="Text">ceiling3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ceiling4">
+ <output-dir compare="Text">ceiling4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="const">
+ <output-dir compare="Text">const</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="degrees">
+ <output-dir compare="Text">degrees</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="divide_double">
+ <output-dir compare="Text">divide_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="divide_float">
+ <output-dir compare="Text">divide_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="divide_int16">
+ <output-dir compare="Text">divide_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="divide_int32">
+ <output-dir compare="Text">divide_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="divide_int64">
+ <output-dir compare="Text">divide_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="divide_int8">
+ <output-dir compare="Text">divide_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="div_mod_case_insensitive">
+ <output-dir compare="Text">div_mod_case_insensitive</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="floor0">
+ <output-dir compare="Text">floor0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="floor1">
+ <output-dir compare="Text">floor1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="floor2">
+ <output-dir compare="Text">floor2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="floor3">
+ <output-dir compare="Text">floor3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="floor4">
+ <output-dir compare="Text">floor4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ifinf">
+ <output-dir compare="Text">ifinf</output-dir>
+ <expected-error>Invalid number of arguments for function if-inf (in line 25, at column 14)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ifnan">
+ <output-dir compare="Text">ifnan</output-dir>
+ <expected-error>Invalid number of arguments for function if-nan (in line 25, at column 14)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ifnanorinf">
+ <output-dir compare="Text">ifnanorinf</output-dir>
+ <expected-error>Invalid number of arguments for function if-nan-or-inf (in line 25, at column 14)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="infinity">
+ <output-dir compare="Text">infinity</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="multiply_double">
+ <output-dir compare="Text">multiply_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="multiply_float">
+ <output-dir compare="Text">multiply_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="multiply_int16">
+ <output-dir compare="Text">multiply_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="multiply_int32">
+ <output-dir compare="Text">multiply_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="multiply_int64">
+ <output-dir compare="Text">multiply_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="multiply_int8">
+ <output-dir compare="Text">multiply_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="power">
+ <output-dir compare="Text">power</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="query-ASTERIXDB-2530">
+ <output-dir compare="Text">query-ASTERIXDB-2530</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="radians">
+ <output-dir compare="Text">radians</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even0">
+ <output-dir compare="Text">round-half-to-even0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even1">
+ <output-dir compare="Text">round-half-to-even1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even2">
+ <output-dir compare="Text">round-half-to-even2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even20">
+ <output-dir compare="Text">round-half-to-even20</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even21">
+ <output-dir compare="Text">round-half-to-even21</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even22">
+ <output-dir compare="Text">round-half-to-even22</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even23">
+ <output-dir compare="Text">round-half-to-even23</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even24">
+ <output-dir compare="Text">round-half-to-even24</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even3">
+ <output-dir compare="Text">round-half-to-even3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even4">
+ <output-dir compare="Text">round-half-to-even4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-to-even5">
+ <output-dir compare="Text">round-half-to-even5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round-half-up">
+ <output-dir compare="Text">round-half-up</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round0">
+ <output-dir compare="Text">round0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round1">
+ <output-dir compare="Text">round1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round2">
+ <output-dir compare="Text">round2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round3">
+ <output-dir compare="Text">round3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round4">
+ <output-dir compare="Text">round4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round5_with_digit_int8">
+ <output-dir compare="Text">round5_with_digit_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round6_with_digit_int16">
+ <output-dir compare="Text">round6_with_digit_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round7_with_digit_int32">
+ <output-dir compare="Text">round7_with_digit_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round8_with_digit_int64">
+ <output-dir compare="Text">round8_with_digit_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round9_with_digit_float">
+ <output-dir compare="Text">round9_with_digit_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round10_with_digit_double">
+ <output-dir compare="Text">round10_with_digit_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="round11_invalid">
+ <output-dir compare="Text">round11_invalid</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="subtract_double">
+ <output-dir compare="Text">subtract_double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="subtract_float">
+ <output-dir compare="Text">subtract_float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="subtract_int16">
+ <output-dir compare="Text">subtract_int16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="subtract_int32">
+ <output-dir compare="Text">subtract_int32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="subtract_int64">
+ <output-dir compare="Text">subtract_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="subtract_int8">
+ <output-dir compare="Text">subtract_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="trunc">
+ <output-dir compare="Text">trunc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="unary-minus_double_02">
+ <output-dir compare="Text">unary-minus_double_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="unary-minus_float_02">
+ <output-dir compare="Text">unary-minus_float_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="unary-minus_int_02">
+ <output-dir compare="Text">unary-minus_int_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="unary-minus_null">
+ <output-dir compare="Text">unary-minus_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="query-issue355"> <!-- @Yingyi, This one fails in the aql tests!! -->
+ <output-dir compare="Text">query-issue355</output-dir>
+ <!-- <expected-error>org.apache.asterix.common.exceptions.AsterixException</expected-error> -->
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="acos">
+ <output-dir compare="Text">acos</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="asin">
+ <output-dir compare="Text">asin</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="atan">
+ <output-dir compare="Text">atan</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="atan2">
+ <output-dir compare="Text">atan2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="cos">
+ <output-dir compare="Text">cos</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="cosh">
+ <output-dir compare="Text">cosh</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="sin">
+ <output-dir compare="Text">sin</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="sinh">
+ <output-dir compare="Text">sinh</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="tan">
+ <output-dir compare="Text">tan</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="tanh">
+ <output-dir compare="Text">tanh</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="exp">
+ <output-dir compare="Text">exp</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="ln">
+ <output-dir compare="Text">ln</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="log">
+ <output-dir compare="Text">log</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="sqrt">
+ <output-dir compare="Text">sqrt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="scientific">
+ <output-dir compare="Text">scientific</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="scientific_error">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier e (in line 24, at column 10)</expected-error>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier e (in line 24, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="sign">
+ <output-dir compare="Text">sign</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="non-finite">
+ <output-dir compare="Clean-JSON">non-finite</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="add_int8">
+ <output-dir compare="Text">add_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="multiply_int8">
+ <output-dir compare="Text">multiply_int8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
+ <compilation-unit name="unary-minus_double_02">
+ <output-dir compare="Text">unary-minus_double_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="open-closed">
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="c2c-w-optional">
+ <output-dir compare="Text">c2c-w-optional</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="c2c-wo-optional">
+ <output-dir compare="Text">c2c-wo-optional</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="c2c">
+ <output-dir compare="Text">c2c</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="open-closed">
+ <compilation-unit name="heterog-list-ordered01">
+ <output-dir compare="Text">heterog-list-ordered01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="heterog-list01">
+ <output-dir compare="Text">heterog-list01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="heterog-list02">
+ <output-dir compare="Text">heterog-list02</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="heterog-list03">
+ <output-dir compare="Text">heterog-list03</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-01">
+ <output-dir compare="Text">open-closed-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-12">
+ <output-dir compare="Text">open-closed-12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-14">
+ <output-dir compare="Text">open-closed-14</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue134">
+ <output-dir compare="Text">query-issue134</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue55">
+ <output-dir compare="Text">query-issue55</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue55-1">
+ <output-dir compare="Text">query-issue55-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue166">
+ <output-dir compare="Text">query-issue166</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue208">
+ <output-dir compare="Text">query-issue208</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue236">
+ <output-dir compare="Text">query-issue236</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-15">
+ <output-dir compare="Text">open-closed-15</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-16">
+ <output-dir compare="Text">open-closed-16</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-17">
+ <output-dir compare="Text">open-closed-17</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-19">
+ <output-dir compare="Text">open-closed-19</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-20">
+ <output-dir compare="Text">open-closed-20</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-21">
+ <output-dir compare="Text">open-closed-21</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-22">
+ <output-dir compare="Text">open-closed-22</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-24">
+ <output-dir compare="Text">open-closed-24</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-25">
+ <output-dir compare="Text">open-closed-25</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-26">
+ <output-dir compare="Text">open-closed-26</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-28">
+ <output-dir compare="Text">open-closed-28</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-29">
+ <output-dir compare="Text">open-closed-29</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-30">
+ <output-dir compare="Text">open-closed-30</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-31">
+ <output-dir compare="Text">open-closed-31</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-32">
+ <output-dir compare="Text">open-closed-32</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="open-closed-33">
+ <output-dir compare="Text">open-closed-33</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-proposal02">
+ <output-dir compare="Text">query-proposal02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-proposal">
+ <output-dir compare="Text">query-proposal</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue350">
+ <output-dir compare="Text">query-issue350</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue350-2">
+ <output-dir compare="Text">query-issue350-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue343">
+ <output-dir compare="Text">query-issue343</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue343-2">
+ <output-dir compare="Text">query-issue343-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue196">
+ <output-dir compare="Text">query-issue196</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue377">
+ <output-dir compare="Text">query-issue377</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue410">
+ <output-dir compare="Text">query-issue410</output-dir>
+ <expected-error>Field type double cannot be promoted to type string</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue453">
+ <output-dir compare="Text">query-issue453</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue453-2">
+ <output-dir compare="Text">query-issue453-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue456">
+ <output-dir compare="Text">query-issue456</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue465">
+ <output-dir compare="Text">query-issue465</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue258">
+ <output-dir compare="Text">query-issue258</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue423">
+ <output-dir compare="Text">query-issue423</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue423-2">
+ <output-dir compare="Text">query-issue423-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue442"><!-- Exception is not thrown!! -->
+ <output-dir compare="Text">query-issue442</output-dir>
+ <!-- <expected-error>org.apache.asterix.common.exceptions.AsterixException</expected-error> -->
+ </compilation-unit>
+ </test-case>
+ <!--<test-case FilePath="open-closed">
+ <compilation-unit name="query-issue487">
+ <output-dir compare="Text">query-issue487</output-dir>
+ <expected-error>org.apache.asterix.common.exceptions.AsterixException</expected-error>
+ </compilation-unit>
+ </test-case> -->
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue592">
+ <output-dir compare="Text">query-issue592</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue625">
+ <output-dir compare="Text">query-issue625</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue208">
+ <output-dir compare="Text">query-issue208</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue236">
+ <output-dir compare="Text">query-issue236</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="quantifiers">
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="any_and_every_01">
+ <output-dir compare="Text">any_and_every_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="anysat_01">
+ <output-dir compare="Text">somesat_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="everysat_01">
+ <output-dir compare="Text">everysat_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="query-ASTERIXDB-1005">
+ <output-dir compare="Text">query-ASTERIXDB-1005</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="query-ASTERIXDB-1674">
+ <output-dir compare="Text">query-ASTERIXDB-1674</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="query-ASTERIXDB-2307">
+ <output-dir compare="Text">query-ASTERIXDB-2307</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="query-ASTERIXDB-2696">
+ <output-dir compare="Text">query-ASTERIXDB-2696</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="query-ASTERIXDB-2947">
+ <output-dir compare="Text">query-ASTERIXDB-2947</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="everysat_02">
+ <output-dir compare="Text">everysat_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="everysat_03">
+ <output-dir compare="Text">everysat_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="everysat_04">
+ <output-dir compare="Text">everysat_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="somesat_01">
+ <output-dir compare="Text">somesat_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="somesat_02">
+ <output-dir compare="Text">somesat_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="somesat_03">
+ <output-dir compare="Text">somesat_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="somesat_04">
+ <output-dir compare="Text">somesat_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="somesat_05">
+ <output-dir compare="Text">somesat_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="quantifiers">
+ <compilation-unit name="somesat_06">
+ <output-dir compare="Text">somesat_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="range-hints">
+ <test-case FilePath="range-hints">
+ <compilation-unit name="order-by">
+ <output-dir compare="Text">order-by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- Fail sporadically <test-case FilePath="range-hints">
+ <compilation-unit name="order-by-exception_01">
+ <output-dir compare="Text">order-by</output-dir>
+ <expected-error>org.json.JSONException: JSONObject['summary'] not found</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="range-hints">
+ <compilation-unit name="order-by-exception_02">
+ <output-dir compare="Text">order-by</output-dir>
+ <expected-error>org.json.JSONException: JSONObject['summary'] not found</expected-error>
+ </compilation-unit>
+ </test-case> -->
+ </test-group>
+ <test-group name="resolution">
+ <test-case FilePath="resolution">
+ <compilation-unit name="conflict-field-dataset">
+ <output-dir compare="Text">conflict-field-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="conflict-field-dataset-from">
+ <output-dir compare="Text">conflict-field-dataset-from</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="conflict-field-dataset-fromterm">
+ <output-dir compare="Text">conflict-field-dataset-from</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="conflict-field-dataset-join">
+ <output-dir compare="Text">conflict-field-dataset-from</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="conflict-field-dataset-quantifier">
+ <output-dir compare="Text">conflict-field-dataset-from</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="conflict-field-dataset-unnest">
+ <output-dir compare="Text">conflict-field-dataset-from</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="conflict-fields-dataset">
+ <output-dir compare="Text">conflict-field-dataset</output-dir>
+ <expected-error>Cannot resolve ambiguous alias reference for identifier samptable</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="fullyqualified">
+ <output-dir compare="Text">fullyqualified</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="fullyqualified2">
+ <output-dir compare="Text">fullyqualified2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="order_1">
+ <output-dir compare="Text">order_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="groupby_rename_with_sugar">
+ <output-dir compare="Text">groupby_rename_with_sugar</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="function_dataverse">
+ <output-dir compare="Text">function_dataverse</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="field_accessor_1">
+ <output-dir compare="Text">field_accessor_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="resolution">
+ <compilation-unit name="field_accessor_2_negative">
+ <output-dir compare="Text">field_accessor_1</output-dir>
+ <expected-error>ASX1074: Cannot resolve ambiguous alias reference for identifier c2 (in line 25, at column 7)</expected-error>
+ <expected-error>ASX1074: Cannot resolve ambiguous alias reference for identifier c2 (in line 25, at column 7)</expected-error>
+ <expected-error>ASX1074: Cannot resolve ambiguous alias reference for identifier c2 (in line 25, at column 7)</expected-error>
+ <expected-error>ASX1074: Cannot resolve ambiguous alias reference for identifier c10 (in line 25, at column 51)</expected-error>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier c20 (in line 26, at column 8)</expected-error>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier c1 (in line 25, at column 19)</expected-error>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier samptable (in line 25, at column 14)</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="scan">
+ <test-case FilePath="scan">
+ <compilation-unit name="10">
+ <output-dir compare="Text">10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="20">
+ <output-dir compare="Text">20</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="issue238_query_1">
+ <output-dir compare="Text">issue238_query_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="issue238_query_2">
+ <output-dir compare="Text">issue238_query_2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- The syntax this test case tets isn't really invalid
+ <test-case FilePath="scan">
+ <compilation-unit name="invalid-scan-syntax">
+ <output-dir compare="Text">invalid-scan-syntax</output-dir>
+ <expected-error>java.lang.IllegalStateException: no result file</expected-error>
+ </compilation-unit>
+ </test-case>-->
+ <test-case FilePath="scan">
+ <compilation-unit name="30">
+ <output-dir compare="Text">30</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="alltypes_01">
+ <output-dir compare="Text">alltypes_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="alltypes_01">
+ <output-dir compare="Clean-JSON">alltypes_01-cleanjson</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="alltypes_01">
+ <output-dir compare="Lossless-JSON">alltypes_01-losslessjson</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="alltypes_01">
+ <output-dir compare="Lossless-ADM-JSON">alltypes_01-losslessadmjson</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="alltypes_02">
+ <output-dir compare="Text">alltypes_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="numeric_types_01">
+ <output-dir compare="Text">numeric_types_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="scan">
+ <compilation-unit name="spatial_types_01">
+ <output-dir compare="Text">spatial_types_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="scan">
+ <compilation-unit name="spatial_types_02">
+ <output-dir compare="Text">spatial_types_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="scan">
+ <compilation-unit name="temp_types_01">
+ <output-dir compare="Text">temp_types_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--
+ <test-case FilePath="scan">
+ <compilation-unit name="temp_types_02">
+ <output-dir compare="Text">temp_types_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ </test-group>
+ <test-group name="select-star">
+ <test-case FilePath="select-star">
+ <compilation-unit name="group_by">
+ <output-dir compare="Text">group_by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="from">
+ <output-dir compare="Text">from</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="join">
+ <output-dir compare="Text">join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="mixed">
+ <output-dir compare="Text">mixed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="no_star">
+ <output-dir compare="Text">no_star</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="var_star">
+ <output-dir compare="Text">var_star</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="var_star_2">
+ <output-dir compare="Text">var_star_2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="semistructured">
+ <test-case FilePath="semistructured">
+ <compilation-unit name="count-nullable">
+ <output-dir compare="Text">count-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="semistructured">
+ <compilation-unit name="cust-filter">
+ <output-dir compare="Text">cust-filter</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="semistructured">
+ <compilation-unit name="has-param1">
+ <output-dir compare="Text">has-param1</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="similarity">
+ <test-case FilePath="similarity">
+ <compilation-unit name="edit-distance-check_ints">
+ <output-dir compare="Text">edit-distance-check_ints</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="edit-distance-check_strings">
+ <output-dir compare="Text">edit-distance-check_strings</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="edit-distance-check_unicode">
+ <output-dir compare="Text">edit-distance-check_unicode</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="edit-distance-list-is-filterable">
+ <output-dir compare="Text">edit-distance-list-is-filterable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="edit-distance-string-is-filterable">
+ <output-dir compare="Text">edit-distance-string-is-filterable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="edit-distance_ints">
+ <output-dir compare="Text">edit-distance_ints</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="edit-distance_strings">
+ <output-dir compare="Text">edit-distance_strings</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="fuzzyeq-edit-distance">
+ <output-dir compare="Text">fuzzyeq-edit-distance</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="fuzzyeq-similarity-jaccard">
+ <output-dir compare="Text">fuzzyeq-similarity-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="prefix-len-jaccard">
+ <output-dir compare="Text">prefix-len-jaccard</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-check_ints">
+ <output-dir compare="Text">similarity-jaccard-check_ints</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-check_query">
+ <output-dir compare="Text">similarity-jaccard-check_query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-check_strings">
+ <output-dir compare="Text">similarity-jaccard-check_strings</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-prefix-check">
+ <output-dir compare="Text">similarity-jaccard-prefix-check</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-prefix">
+ <output-dir compare="Text">similarity-jaccard-prefix</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-sorted-check_ints">
+ <output-dir compare="Text">similarity-jaccard-sorted-check_ints</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-sorted-check_query">
+ <output-dir compare="Text">similarity-jaccard-sorted-check_query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-sorted-check_strings">
+ <output-dir compare="Text">similarity-jaccard-sorted-check_strings</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-sorted_ints">
+ <output-dir compare="Text">similarity-jaccard-sorted_ints</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-sorted_query">
+ <output-dir compare="Text">similarity-jaccard-sorted_query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-sorted_strings">
+ <output-dir compare="Text">similarity-jaccard-sorted_strings</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard_ints">
+ <output-dir compare="Text">similarity-jaccard_ints</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard_query">
+ <output-dir compare="Text">similarity-jaccard_query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard_strings">
+ <output-dir compare="Text">similarity-jaccard_strings</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="similarity">
+ <compilation-unit name="similarity-jaccard-check_strings_issue628">
+ <output-dir compare="Text">similarity-jaccard-check_strings_issue628</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="spatial">
+ <test-case FilePath="spatial">
+ <compilation-unit name="cell-aggregation-with-filtering">
+ <output-dir compare="Text">cell-aggregation-with-filtering</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="cell-aggregation">
+ <output-dir compare="Text">cell-aggregation</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="circle_accessor">
+ <output-dir compare="Text">circle_accessor</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="circle-intersect-circle">
+ <output-dir compare="Text">circle-intersect-circle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="create-rtree-index">
+ <output-dir compare="Text">create-rtree-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="distance-between-points">
+ <output-dir compare="Text">distance-between-points</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="hints_spatial_partitioning">
+ <output-dir compare="Text">hints_spatial_partitioning</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="line_accessor">
+ <output-dir compare="Text">line_accessor</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="line-intersect-circle">
+ <output-dir compare="Text">line-intersect-circle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="line-intersect-line">
+ <output-dir compare="Text">line-intersect-line</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="line-intersect-polygon">
+ <output-dir compare="Text">line-intersect-polygon</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="line-intersect-rectangle">
+ <output-dir compare="Text">line-intersect-rectangle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="point_accessor">
+ <output-dir compare="Text">point_accessor</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="point-equals-point">
+ <output-dir compare="Text">point-equals-point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="point-in-circle">
+ <output-dir compare="Text">point-in-circle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="point-in-polygon">
+ <output-dir compare="Text">point-in-polygon</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="point-in-rectangle">
+ <output-dir compare="Text">point-in-rectangle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="point-on-line">
+ <output-dir compare="Text">point-on-line</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="polygon_accessor">
+ <output-dir compare="Text">polygon_accessor</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="polygon-intersect-circle">
+ <output-dir compare="Text">polygon-intersect-circle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="polygon-intersect-polygon">
+ <output-dir compare="Text">polygon-intersect-polygon</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="polygon-intersect-rectangle">
+ <output-dir compare="Text">polygon-intersect-rectangle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="rectangle_accessor">
+ <output-dir compare="Text">rectangle_accessor</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="rectangle-intersect-circle">
+ <output-dir compare="Text">rectangle-intersect-circle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="rectangle-intersect-rectangle">
+ <output-dir compare="Text">rectangle-intersect-rectangle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="spatial-area">
+ <output-dir compare="Text">spatial-area</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="spatial-large-data">
+ <output-dir compare="Text">spatial-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="spatial_join_dynamic_partitioning">
+ <output-dir compare="Text">spatial_join_dynamic_partitioning</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="spatial_join_static_partitioning">
+ <output-dir compare="Text">spatial_join_static_partitioning</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="spatial_left_outer_join_st_intersects">
+ <output-dir compare="Text">spatial_left_outer_join_st_intersects</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="spatial">
+ <compilation-unit name="spatial_join_projection_check">
+ <output-dir compare="Text">spatial_join_projection_check</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="sql-compat">
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="in_non_list_01">
+ <output-dir compare="Text">in_non_list_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="outer_join_01">
+ <output-dir compare="Text">outer_join_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="outer_unnest_01">
+ <output-dir compare="Text">outer_unnest_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="select_star_01">
+ <output-dir compare="Text">select_star_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="select_star_02_negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1168: Ambiguous projection in SELECT clause (in line 32, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_01_scalar">
+ <output-dir compare="Text">subquery_coercion_01_scalar</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_02_scalar_negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1169: Unable to do subquery coercion. Subquery returns more than one field (in line 32, at column 29)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Unsupported projection kind (in line 32, at column 20)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Both SELECT and SELECT VALUE are present (in line 35, at column 4)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. (in line 34, at column 3)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_03_cmp">
+ <output-dir compare="Text">subquery_coercion_03_cmp</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_04_cmp_negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1169: Unable to do subquery coercion. Subquery returns more than one field (in line 35, at column 26)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Subquery returns more than one field (in line 35, at column 21)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Both SELECT and SELECT VALUE are present (in line 37, at column 3)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Both SELECT and SELECT VALUE are present (in line 38, at column 3)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. (in line 36, at column 3)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Unsupported projection kind (in line 35, at column 20)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Unsupported projection kind (in line 36, at column 11)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Unsupported projection kind (in line 37, at column 11)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_05_in">
+ <output-dir compare="Text">subquery_coercion_05_in</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_06_in_negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1169: Unable to do subquery coercion. Subquery returns more than one field (in line 36, at column 27)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Subquery returns more than one field (in line 37, at column 17)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Both SELECT and SELECT VALUE are present (in line 38, at column 3)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Both SELECT and SELECT VALUE are present (in line 39, at column 3)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. (in line 37, at column 3)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Unsupported projection kind (in line 36, at column 21)</expected-error>
+ <expected-error>ASX1169: Unable to do subquery coercion. Unsupported projection kind (in line 37, at column 25)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_07_from">
+ <output-dir compare="Text">subquery_coercion_07_from</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="subquery_coercion_08_misc">
+ <output-dir compare="Text">subquery_coercion_08_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="union_all_01">
+ <output-dir compare="Text">union_all_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="sql-compat">
+ <compilation-unit name="union_all_02_negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1171: Unable to process UNION clause. Both SELECT and SELECT VALUE are present (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1171: Unable to process UNION clause. Unequal number of input fields (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1171: Unable to process UNION clause. Unsupported projection kind (in line 29, at column 8)</expected-error>
+ <expected-error>ASX1171: Unable to process UNION clause. Unsupported projection kind (in line 30, at column 8)</expected-error>
+ <expected-error>ASX1171: Unable to process UNION clause. Unsupported projection kind (in line 31, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="statement-params">
+ <!--test-case FilePath="statement-params">
+ <compilation-unit name="index_01">
+ <output-dir compare="Text">index_01</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="statement-params">
+ <compilation-unit name="mixed_01">
+ <output-dir compare="Text">mixed_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="named_01">
+ <output-dir compare="Text">named_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="named_02">
+ <output-dir compare="Text">named_01</output-dir>
+ <expected-error>ASX1086: No value for parameter: $p2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="named_03">
+ <output-dir compare="Text">named_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="named_04">
+ <output-dir compare="Text">named_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="positional_01">
+ <output-dir compare="Text">positional_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="positional_02">
+ <output-dir compare="Text">positional_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="positional_03">
+ <output-dir compare="Text">positional_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="positional_04">
+ <output-dir compare="Text">positional_02</output-dir>
+ <expected-error>ASX1086: No value for parameter: $2</expected-error>
+ <expected-error>ASX1086: No value for parameter: $3</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="positional_05">
+ <output-dir compare="Text">positional_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="statement-params">
+ <compilation-unit name="query-ASTERIXDB-2413">
+ <output-dir compare="Text">query-ASTERIXDB-2413</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="string">
+ <test-case FilePath="string">
+ <compilation-unit name="codepoint-to-string1">
+ <output-dir compare="Text">codepoint-to-string1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="codepoint-to-string2">
+ <output-dir compare="Text">codepoint-to-string2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/001">
+ <output-dir compare="Text">concat/001</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/002">
+ <output-dir compare="Text">concat/002</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string" check-warnings="true">
+ <compilation-unit name="concat/003">
+ <output-dir compare="Text">concat/003</output-dir>
+ <expected-warn>Type mismatch: function string-concat expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 26, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/004">
+ <output-dir compare="Text">concat/004</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/005">
+ <output-dir compare="Text">concat/005</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/006">
+ <output-dir compare="Text">concat/006</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/007">
+ <output-dir compare="Text">concat/007</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/008">
+ <output-dir compare="Text">concat/002</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/concat_pipe">
+ <output-dir compare="Text">concat/002</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="concat/concat_pipe_multi">
+ <output-dir compare="Text">concat/concat_pipe_multi</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="constructor">
+ <output-dir compare="Text">constructor</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="contains_01">
+ <output-dir compare="Text">contains_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="cpttostr01">
+ <output-dir compare="Text">cpttostr01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="cpttostr02">
+ <output-dir compare="Text">cpttostr02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="cpttostr04">
+ <output-dir compare="Text">cpttostr04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="ends-with1">
+ <output-dir compare="Text">ends-with1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="ends-with2">
+ <output-dir compare="Text">ends-with2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="ends-with3">
+ <output-dir compare="Text">ends-with3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="ends-with4">
+ <output-dir compare="Text">ends-with4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="ends-with5">
+ <output-dir compare="Text">ends-with5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="escapes01">
+ <output-dir compare="Text">escapes01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="escapes02">
+ <output-dir compare="Text">escapes02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="initcap">
+ <output-dir compare="Text">initcap</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="join/000">
+ <output-dir compare="Text">join/000</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string" check-warnings="true">
+ <compilation-unit name="join/001">
+ <output-dir compare="Text">join/001</output-dir>
+ <expected-warn>Type mismatch: function string-join expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 24, at column 15)</expected-warn>
+ <expected-warn>Type mismatch: function string-join expects its 1st input parameter to be of type array, but the actual input type is bigint (in line 24, at column 15)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="join/002">
+ <output-dir compare="Text">join/002</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string" check-warnings="true">
+ <compilation-unit name="join/003">
+ <output-dir compare="Text">join/003</output-dir>
+ <expected-warn>Type mismatch: function string-join expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 23, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-join expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 30, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-join expects its 1st input parameter to be of type array, but the actual input type is bigint (in line 33, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-join expects its 1st input parameter to be of type array, but the actual input type is bigint (in line 26, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string" check-warnings="true">
+ <compilation-unit name="join/004">
+ <output-dir compare="Text">join/004</output-dir>
+ <expected-warn>Type mismatch: function string-join expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 26, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string" check-warnings="true">
+ <compilation-unit name="join/005">
+ <output-dir compare="Text">join/005</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="length_01">
+ <output-dir compare="Text">length_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="length_02">
+ <output-dir compare="Text">length_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="length_multi_code_point_01">
+ <output-dir compare="Text">length_multi_code_point_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="like_01">
+ <output-dir compare="Text">like_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="like_02">
+ <output-dir compare="Text">like_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="like_03_negative">
+ <output-dir compare="Text">like_03_negative</output-dir>
+ <expected-error>Invalid pattern '__\c' for LIKE (in line 21, at column 11)</expected-error>
+ <expected-error>Invalid pattern '%\' for LIKE (in line 21, at column 18)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="like_null">
+ <output-dir compare="Text">like_null</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="lowercase">
+ <output-dir compare="Text">lowercase</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches02">
+ <output-dir compare="Text">matches02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches03">
+ <output-dir compare="Text">matches03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches04">
+ <output-dir compare="Text">matches04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches05">
+ <output-dir compare="Text">matches05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches06">
+ <output-dir compare="Text">matches06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches1">
+ <output-dir compare="Text">matches1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches11">
+ <output-dir compare="Text">matches11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches2">
+ <output-dir compare="Text">matches2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches21">
+ <output-dir compare="Text">matches21</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches22">
+ <output-dir compare="Text">matches22</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches23">
+ <output-dir compare="Text">matches23</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matches3">
+ <output-dir compare="Text">matches3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="matchesnull">
+ <output-dir compare="Text">matchesnull</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset0/position">
+ <output-dir compare="Text">position/offset0/position</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset0/pos">
+ <output-dir compare="Text">position/offset0/pos</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset0/position0">
+ <output-dir compare="Text">position/offset0/position0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset0/pos0">
+ <output-dir compare="Text">position/offset0/pos0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset0/pos0_multi_code_point">
+ <output-dir compare="Text">position/offset0/pos0_multi_code_point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset1/position1">
+ <output-dir compare="Text">position/offset1/position1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset1/pos1">
+ <output-dir compare="Text">position/offset1/pos1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="position/offset1/pos1_multi_code_point">
+ <output-dir compare="Text">position/offset1/pos1_multi_code_point</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains/regexp_contains">
+ <output-dir compare="Text">regexp_contains/regexp_contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains/regex_contains">
+ <output-dir compare="Text">regexp_contains/regex_contains</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains/contains_regexp">
+ <output-dir compare="Text">regexp_contains/contains_regexp</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains/contains_regex">
+ <output-dir compare="Text">regexp_contains/contains_regex</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains_with_flag/regexp_contains_with_flag">
+ <output-dir compare="Text">regexp_contains_with_flag/regexp_contains_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains_with_flag/regex_contains_with_flag">
+ <output-dir compare="Text">regexp_contains_with_flag/regex_contains_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains_with_flag/contains_regexp_with_flag">
+ <output-dir compare="Text">regexp_contains_with_flag/contains_regexp_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_contains_with_flag/contains_regex_with_flag">
+ <output-dir compare="Text">regexp_contains_with_flag/contains_regex_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_like/regexp_like">
+ <output-dir compare="Text">regexp_like/regexp_like</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_like/regex_like">
+ <output-dir compare="Text">regexp_like/regex_like</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_like_with_flag/regexp_like_with_flag">
+ <output-dir compare="Text">regexp_like_with_flag/regexp_like_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_like_with_flag/regex_like_with_flag">
+ <output-dir compare="Text">regexp_like_with_flag/regex_like_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_matches/001">
+ <output-dir compare="Text">regexp_matches/001</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_matches/002">
+ <output-dir compare="Text">regexp_matches/002</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_matches/003">
+ <output-dir compare="Text">regexp_matches/003</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_matches/004">
+ <output-dir compare="Text">regexp_matches/004</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regexp_position">
+ <output-dir compare="Text">regexp_position/offset0/regexp_position</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regexp_pos">
+ <output-dir compare="Text">regexp_position/offset0/regexp_pos</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regexp_position0">
+ <output-dir compare="Text">regexp_position/offset0/regexp_position0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regexp_pos0">
+ <output-dir compare="Text">regexp_position/offset0/regexp_pos0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regex_position">
+ <output-dir compare="Text">regexp_position/offset0/regex_position</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regex_pos">
+ <output-dir compare="Text">regexp_position/offset0/regex_pos</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regex_position0">
+ <output-dir compare="Text">regexp_position/offset0/regex_position0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset0/regex_pos0">
+ <output-dir compare="Text">regexp_position/offset0/regex_pos0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset1/regexp_position1">
+ <output-dir compare="Text">regexp_position/offset1/regexp_position1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset1/regexp_pos1">
+ <output-dir compare="Text">regexp_position/offset1/regexp_pos1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset1/regex_position1">
+ <output-dir compare="Text">regexp_position/offset1/regex_position1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position/offset1/regex_pos1">
+ <output-dir compare="Text">regexp_position/offset1/regex_pos1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regexp_position_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regexp_position_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regexp_pos_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regexp_pos_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regexp_position0_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regexp_position0_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regexp_pos0_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regexp_pos0_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regex_position_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regex_position_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regex_pos_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regex_pos_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr-ASTERIXDB-2949">
+ <output-dir compare="Text">substr-ASTERIXDB-2949</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regex_position0_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regex_position0_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset0/regex_pos0_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset0/regex_pos0_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset1/regexp_position1_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset1/regexp_position1_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset1/regexp_pos1_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset1/regexp_pos1_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset1/regex_position1_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset1/regex_position1_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_position_with_flag/offset1/regex_pos1_with_flag">
+ <output-dir compare="Text">regexp_position_with_flag/offset1/regex_pos1_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_split/001">
+ <output-dir compare="Text">regexp_split/001</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_split/002">
+ <output-dir compare="Text">regexp_split/002</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_split/003">
+ <output-dir compare="Text">regexp_split/003</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_split/004">
+ <output-dir compare="Text">regexp_split/004</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="repeat">
+ <output-dir compare="Text">repeat</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="replace">
+ <output-dir compare="Text">replace</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="replace_with_limit">
+ <output-dir compare="Text">replace_with_limit</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_replace/regexp_replace">
+ <output-dir compare="Text">regexp_replace/regexp_replace</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_replace/regex_replace">
+ <output-dir compare="Text">regexp_replace/regex_replace</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_replace_with_flag/regexp_replace_with_flag">
+ <output-dir compare="Text">regexp_replace_with_flag/regexp_replace_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="regexp_replace_with_flag/regex_replace_with_flag">
+ <output-dir compare="Text">regexp_replace_with_flag/regex_replace_with_flag</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="reverse">
+ <output-dir compare="Text">reverse</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="split">
+ <output-dir compare="Text">split</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="starts-with1">
+ <output-dir compare="Text">starts-with1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="starts-with2">
+ <output-dir compare="Text">starts-with2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="starts-with3">
+ <output-dir compare="Text">starts-with3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="starts-with4">
+ <output-dir compare="Text">starts-with4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="starts-with5">
+ <output-dir compare="Text">starts-with5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-equal1">
+ <output-dir compare="Text">string-equal1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-equal2">
+ <output-dir compare="Text">string-equal2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-equal3">
+ <output-dir compare="Text">string-equal3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-equal4">
+ <output-dir compare="Text">string-equal4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-literal1">
+ <output-dir compare="Text">string-literal1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-to-codepoint">
+ <output-dir compare="Text">string-to-codepoint</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-to-codepoint1">
+ <output-dir compare="Text">string-to-codepoint1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-to-codepoint2">
+ <output-dir compare="Text">string-to-codepoint2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string_to_codepoint_multi_codepoints_01">
+ <output-dir compare="Text">string_to_codepoint_multi_codepoints_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="strlen02">
+ <output-dir compare="Text">strlen02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="strlen03">
+ <output-dir compare="Text">strlen03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="strtocpt01">
+ <output-dir compare="Text">strtocpt01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="strtocpt02">
+ <output-dir compare="Text">strtocpt02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="strtocpt03">
+ <output-dir compare="Text">strtocpt03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr01/offset0/substring">
+ <output-dir compare="Text">substr01/offset0/substring</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr01/offset0/substr">
+ <output-dir compare="Text">substr01/offset0/substr</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr01/offset0/substring0">
+ <output-dir compare="Text">substr01/offset0/substring0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr01/offset0/substr0">
+ <output-dir compare="Text">substr01/offset0/substr0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr01/offset1/substring1">
+ <output-dir compare="Text">substr01/offset1/substring1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr01/offset1/substr1">
+ <output-dir compare="Text">substr01/offset1/substr1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr04">
+ <output-dir compare="Text">substr04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr05">
+ <output-dir compare="Text">substr05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substr06">
+ <output-dir compare="Text">substr06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring_multi_codepoint_01">
+ <output-dir compare="Text">substring_multi_codepoint_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-after-1">
+ <output-dir compare="Text">substring-after-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-after-2">
+ <output-dir compare="Text">substring-after-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-after-3">
+ <output-dir compare="Text">substring-after-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-after-4">
+ <output-dir compare="Text">substring-after-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-before-1">
+ <output-dir compare="Text">substring-before-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-before-2">
+ <output-dir compare="Text">substring-before-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-before-3">
+ <output-dir compare="Text">substring-before-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-1/offset0/substring">
+ <output-dir compare="Text">substring2-1/offset0/substring</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-1/offset0/substr">
+ <output-dir compare="Text">substring2-1/offset0/substr</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-1/offset0/substring0">
+ <output-dir compare="Text">substring2-1/offset0/substring0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-1/offset0/substr0">
+ <output-dir compare="Text">substring2-1/offset0/substr0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-1/offset1/substring1">
+ <output-dir compare="Text">substring2-1/offset1/substring1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-1/offset1/substr1">
+ <output-dir compare="Text">substring2-1/offset1/substr1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-2">
+ <output-dir compare="Text">substring2-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-3">
+ <output-dir compare="Text">substring2-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring2-4">
+ <output-dir compare="Text">substring2-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring_01">
+ <output-dir compare="Text">substring_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="title">
+ <output-dir compare="Text">initcap</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="toLowerCase02">
+ <output-dir compare="Text">toLowerCase02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="toLowerCase03">
+ <output-dir compare="Text">toLowerCase03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="toLowerCase04">
+ <output-dir compare="Text">toLowerCase04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="trim">
+ <output-dir compare="Text">trim</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="ltrim">
+ <output-dir compare="Text">ltrim</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="rtrim">
+ <output-dir compare="Text">rtrim</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="uppercase">
+ <output-dir compare="Text">uppercase</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="varlen-encoding">
+ <output-dir compare="Text">varlen-encoding</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="utf8">
+ <output-dir compare="Text">utf8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="query-ASTERIXDB-1190">
+ <output-dir compare="Text">query-ASTERIXDB-1190</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="string-equal-public">
+ <output-dir compare="Text">string-equal-public</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-after-5">
+ <output-dir compare="Text">substring-after-5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
+ <compilation-unit name="substring-after-6">
+ <output-dir compare="Text">substring-after-5</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="subquery">
+ <test-case FilePath="subquery">
+ <compilation-unit name="aggregate_join">
+ <output-dir compare="Text">aggregate_join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="aggregate_join_external">
+ <output-dir compare="Text">aggregate_join_external</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="gby_inline">
+ <output-dir compare="Text">gby_inline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1168">
+ <output-dir compare="Text">query-ASTERIXDB-1168</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="decorrelate_with_unique_id">
+ <output-dir compare="Text">decorrelate_with_unique_id</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="exists">
+ <output-dir compare="Text">exists</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="in">
+ <output-dir compare="Text">in</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="in_as_or">
+ <output-dir compare="Text">in_as_or</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="in_correlated">
+ <output-dir compare="Text">in</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="in_let">
+ <output-dir compare="Text">in_let</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="not_exists">
+ <output-dir compare="Text">not_exists</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="not_in">
+ <output-dir compare="Text">not_in</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="alias_negative">
+ <output-dir compare="Text">alias_negative</output-dir>
+ <expected-error>Need an alias for the enclosed expression</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="relational_division">
+ <output-dir compare="Text">relational_division</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="relational_division2">
+ <output-dir compare="Text">relational_division</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="relational_division3">
+ <output-dir compare="Text">relational_division</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="non_unary_subplan_01">
+ <output-dir compare="Text">non_unary_subplan_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1571">
+ <output-dir compare="Text">query-ASTERIXDB-1571</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1571-2">
+ <output-dir compare="Text">query-ASTERIXDB-1571</output-dir>
+ <expected-error>Need an alias for the enclosed expression</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1571-3">
+ <output-dir compare="Text">query-ASTERIXDB-1571-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1571-4">
+ <output-dir compare="Text">query-ASTERIXDB-1571-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1572">
+ <output-dir compare="Text">query-ASTERIXDB-1572</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1574">
+ <output-dir compare="Text">query-ASTERIXDB-1574</output-dir>
+ <expected-error>ASX1091: Type mismatch: expected value of type multiset or array, but got the value of type object (in line 25, at column 41)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1574-2">
+ <output-dir compare="Text">query-ASTERIXDB-1574</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1574-3">
+ <output-dir compare="Text">query-ASTERIXDB-1574</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1597">
+ <output-dir compare="Text">query-ASTERIXDB-1597</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-1674">
+ <output-dir compare="Text">query-ASTERIXDB-1674</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-2815">
+ <output-dir compare="Text">query-ASTERIXDB-2815</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-2845">
+ <output-dir compare="Text">query-ASTERIXDB-2845</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-3006">
+ <output-dir compare="Text">query-ASTERIXDB-3006</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="subset-collection">
+ <test-case FilePath="subset-collection">
+ <compilation-unit name="01">
+ <output-dir compare="Text">01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subset-collection">
+ <compilation-unit name="02">
+ <output-dir compare="Text">02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subset-collection">
+ <compilation-unit name="03">
+ <output-dir compare="Text">03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subset-collection">
+ <compilation-unit name="05">
+ <output-dir compare="Text">05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subset-collection">
+ <compilation-unit name="06">
+ <output-dir compare="Text">06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="subset-collection">
+ <compilation-unit name="07">
+ <output-dir compare="Text">07</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="synonym">
+ <test-case FilePath="synonym">
+ <compilation-unit name="synonym-01">
+ <output-dir compare="Text">synonym-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="synonym">
+ <compilation-unit name="synonym-02-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1063: Cannot find dataverse with name UNKNOWN_DATAVERSE</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="synonym">
+ <compilation-unit name="synonym-03">
+ <output-dir compare="Text">synonym-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tokenizers">
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="counthashed-gram-tokens_01">
+ <output-dir compare="Text">counthashed-gram-tokens_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="counthashed-gram-tokens_02">
+ <output-dir compare="Text">counthashed-gram-tokens_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="counthashed-word-tokens_01">
+ <output-dir compare="Text">counthashed-word-tokens_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="gram-tokens_01">
+ <output-dir compare="Text">gram-tokens_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="gram-tokens_02">
+ <output-dir compare="Text">gram-tokens_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="hashed-gram-tokens_01">
+ <output-dir compare="Text">hashed-gram-tokens_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="hashed-gram-tokens_02">
+ <output-dir compare="Text">hashed-gram-tokens_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="hashed-word-tokens_01">
+ <output-dir compare="Text">hashed-word-tokens_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="word-tokens_01">
+ <output-dir compare="Text">word-tokens_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tokenizers">
+ <compilation-unit name="word-tokens_02">
+ <output-dir compare="Text">word-tokens_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tpcds">
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1580">
+ <output-dir compare="Text">query-ASTERIXDB-1580</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581">
+ <output-dir compare="Text">query-ASTERIXDB-1581</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581-2">
+ <output-dir compare="Text">query-ASTERIXDB-1581-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581-3">
+ <output-dir compare="Text">query-ASTERIXDB-1581-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581-4">
+ <output-dir compare="Text">query-ASTERIXDB-1581-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581-5">
+ <output-dir compare="Text">query-ASTERIXDB-1581-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581-6">
+ <output-dir compare="Text">query-ASTERIXDB-1581-6</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581-correlated">
+ <output-dir compare="Text">query-ASTERIXDB-1581-correlated</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1581-correlated-2">
+ <output-dir compare="Text">query-ASTERIXDB-1581-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1591">
+ <output-dir compare="Text">query-ASTERIXDB-1591</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1596">
+ <output-dir compare="Text">query-ASTERIXDB-1596</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="query-ASTERIXDB-1602">
+ <output-dir compare="Text">query-ASTERIXDB-1602</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q01">
+ <output-dir compare="Text">q01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q03">
+ <output-dir compare="Text">q03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q05">
+ <output-dir compare="Text">q05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q07">
+ <output-dir compare="Text">q07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q09">
+ <output-dir compare="Text">q09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q12">
+ <output-dir compare="Text">q12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q15">
+ <output-dir compare="Text">q15</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q18">
+ <output-dir compare="Text">q18</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q19">
+ <output-dir compare="Text">q19</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q20">
+ <output-dir compare="Text">q20</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q21">
+ <output-dir compare="Text">q21</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q22">
+ <output-dir compare="Text">q22</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q24a">
+ <output-dir compare="Text">q24a</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q24b">
+ <output-dir compare="Text">q24b</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q25">
+ <output-dir compare="Text">q25</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q26">
+ <output-dir compare="Text">q26</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q27">
+ <output-dir compare="Text">q27</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q29">
+ <output-dir compare="Text">q29</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q30">
+ <output-dir compare="Text">q30</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q31">
+ <output-dir compare="Text">q31</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q34">
+ <output-dir compare="Text">q34</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q36">
+ <output-dir compare="Text">q36</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q37">
+ <output-dir compare="Text">q37</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q41">
+ <output-dir compare="Text">q41</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q42">
+ <output-dir compare="Text">q42</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q43">
+ <output-dir compare="Text">q43</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q45">
+ <output-dir compare="Text">q45</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q46">
+ <output-dir compare="Text">q46</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q49">
+ <output-dir compare="Text">q49</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q50">
+ <output-dir compare="Text">q50</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q52">
+ <output-dir compare="Text">q52</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q55">
+ <output-dir compare="Text">q55</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q57">
+ <output-dir compare="Text">q57</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q59">
+ <output-dir compare="Text">q59</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q62">
+ <output-dir compare="Text">q62</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q63">
+ <output-dir compare="Text">q63</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q67">
+ <output-dir compare="Text">q67</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q68">
+ <output-dir compare="Text">q68</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q70">
+ <output-dir compare="Text">q70</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q73">
+ <output-dir compare="Text">q73</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q77">
+ <output-dir compare="Text">q77</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q79">
+ <output-dir compare="Text">q79</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q80">
+ <output-dir compare="Text">q80</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q81">
+ <output-dir compare="Text">q81</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q82">
+ <output-dir compare="Text">q82</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q85">
+ <output-dir compare="Text">q85</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q86">
+ <output-dir compare="Text">q86</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q88">
+ <output-dir compare="Text">q88</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q89">
+ <output-dir compare="Text">q89</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q90">
+ <output-dir compare="Text">q90</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q91">
+ <output-dir compare="Text">q91</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q92">
+ <output-dir compare="Text">q92</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q94">
+ <output-dir compare="Text">q94</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q95">
+ <output-dir compare="Text">q95</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q96">
+ <output-dir compare="Text">q96</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="q98">
+ <output-dir compare="Text">q98</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tpch">
+ <test-case FilePath="tpch">
+ <compilation-unit name="distinct_by">
+ <output-dir compare="Text">distinct_by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="group_no_agg">
+ <output-dir compare="Text">group_no_agg</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="nest_aggregate">
+ <output-dir compare="Text">nest_aggregate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="nest_aggregate2">
+ <output-dir compare="Text">nest_aggregate2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue638">
+ <output-dir compare="Text">query-issue638</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue785">
+ <output-dir compare="Text">query-issue785</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue785-2">
+ <output-dir compare="Text">query-issue785-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue786">
+ <output-dir compare="Text">query-issue786</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue601">
+ <output-dir compare="Text">query-issue601</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q10_returned_item">
+ <output-dir compare="Text">q10_returned_item</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q10_returned_item_int64">
+ <output-dir compare="Text">q10_returned_item_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q11_important_stock">
+ <output-dir compare="Text">q11_important_stock</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q12_shipping">
+ <output-dir compare="Text">q12_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q13_customer_distribution">
+ <output-dir compare="Text">q13_customer_distribution</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q14_promotion_effect">
+ <output-dir compare="Text">q14_promotion_effect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q15_top_supplier">
+ <output-dir compare="Text">q15_top_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q16_parts_supplier_relationship">
+ <output-dir compare="Text">q16_parts_supplier_relationship</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q17_small_quantity_order_revenue">
+ <output-dir compare="Text">q17_small_quantity_order_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q17_large_gby_variant">
+ <output-dir compare="Text">q17_large_gby_variant</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q18_large_volume_customer">
+ <output-dir compare="Text">q18_large_volume_customer</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q19_discounted_revenue">
+ <output-dir compare="Text">q19_discounted_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q01_pricing_summary_report_nt">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q20_potential_part_promotion">
+ <output-dir compare="Text">q20_potential_part_promotion</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q21_suppliers_who_kept_orders_waiting">
+ <output-dir compare="Text">q21_suppliers_who_kept_orders_waiting</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q22_global_sales_opportunity">
+ <output-dir compare="Text">q22_global_sales_opportunity</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q02_minimum_cost_supplier">
+ <output-dir compare="Text">q02_minimum_cost_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q03_shipping_priority_nt">
+ <output-dir compare="Text">q03_shipping_priority_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q04_order_priority">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q05_local_supplier_volume">
+ <output-dir compare="Text">q05_local_supplier_volume</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q06_forecast_revenue_change">
+ <output-dir compare="Text">q06_forecast_revenue_change</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q07_volume_shipping">
+ <output-dir compare="Text">q07_volume_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q08_national_market_share">
+ <output-dir compare="Text">q08_national_market_share</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q09_product_type_profit_nt">
+ <output-dir compare="Text">q09_product_type_profit_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue562">
+ <output-dir compare="Text">query-issue562</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue810">
+ <output-dir compare="Text">query-issue810</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue810-2">
+ <output-dir compare="Text">query-issue810-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue810-3">
+ <output-dir compare="Text">query-issue810-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue827">
+ <output-dir compare="Text">query-issue827</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="query-issue827-2">
+ <output-dir compare="Text">query-issue827-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch">
+ <compilation-unit name="q01-ASTERIXDB-830">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tpch-sql">
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="distinct_by">
+ <output-dir compare="Text">distinct_by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="group_no_agg">
+ <output-dir compare="Text">group_no_agg</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="nest_aggregate">
+ <output-dir compare="Text">nest_aggregate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="nest_aggregate2">
+ <output-dir compare="Text">nest_aggregate2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue638">
+ <output-dir compare="Text">query-issue638</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue785">
+ <output-dir compare="Text">query-issue785</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue785-2">
+ <output-dir compare="Text">query-issue785-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue786">
+ <output-dir compare="Text">query-issue786</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue601">
+ <output-dir compare="Text">query-issue601</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q10_returned_item">
+ <output-dir compare="Text">q10_returned_item</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q10_returned_item_int64">
+ <output-dir compare="Text">q10_returned_item_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q11_important_stock">
+ <output-dir compare="Text">q11_important_stock</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q12_shipping">
+ <output-dir compare="Text">q12_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q13_customer_distribution">
+ <output-dir compare="Text">q13_customer_distribution</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q14_promotion_effect">
+ <output-dir compare="Text">q14_promotion_effect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q15_top_supplier">
+ <output-dir compare="Text">q15_top_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q16_parts_supplier_relationship">
+ <output-dir compare="Text">q16_parts_supplier_relationship</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q17_small_quantity_order_revenue">
+ <output-dir compare="Text">q17_small_quantity_order_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q17_large_gby_variant">
+ <output-dir compare="Text">q17_large_gby_variant</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q18_large_volume_customer">
+ <output-dir compare="Text">q18_large_volume_customer</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q19_discounted_revenue">
+ <output-dir compare="Text">q19_discounted_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q01_pricing_summary_report_nt">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q20_potential_part_promotion">
+ <output-dir compare="Text">q20_potential_part_promotion</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q21_suppliers_who_kept_orders_waiting">
+ <output-dir compare="Text">q21_suppliers_who_kept_orders_waiting</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q22_global_sales_opportunity">
+ <output-dir compare="Text">q22_global_sales_opportunity</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q02_minimum_cost_supplier">
+ <output-dir compare="Text">q02_minimum_cost_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q03_shipping_priority_nt">
+ <output-dir compare="Text">q03_shipping_priority_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q04_order_priority">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q05_local_supplier_volume">
+ <output-dir compare="Text">q05_local_supplier_volume</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q06_forecast_revenue_change">
+ <output-dir compare="Text">q06_forecast_revenue_change</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q07_volume_shipping">
+ <output-dir compare="Text">q07_volume_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q08_national_market_share">
+ <output-dir compare="Text">q08_national_market_share</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="q09_product_type_profit_nt">
+ <output-dir compare="Text">q09_product_type_profit_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue562">
+ <output-dir compare="Text">query-issue562</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue810">
+ <output-dir compare="Text">query-issue810</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue810-2">
+ <output-dir compare="Text">query-issue810-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql">
+ <compilation-unit name="query-issue810-3">
+ <output-dir compare="Text">query-issue810-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tpch-sql-sugar">
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="distinct_by">
+ <output-dir compare="Text">distinct_by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="group_no_agg">
+ <output-dir compare="Text">group_no_agg</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="nest_aggregate">
+ <output-dir compare="Text">nest_aggregate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="nest_aggregate2">
+ <output-dir compare="Text">nest_aggregate2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue638">
+ <output-dir compare="Text">query-issue638</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue785">
+ <output-dir compare="Text">query-issue785</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue785-2">
+ <output-dir compare="Text">query-issue785-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue786">
+ <output-dir compare="Text">query-issue786</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue601">
+ <output-dir compare="Text">query-issue601</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q10_returned_item">
+ <output-dir compare="Text">q10_returned_item</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q10_returned_item_int64">
+ <output-dir compare="Text">q10_returned_item_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q11_important_stock">
+ <output-dir compare="Text">q11_important_stock</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q12_shipping">
+ <output-dir compare="Text">q12_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q12_shipping_broadcast">
+ <output-dir compare="Text">q12_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q13_customer_distribution">
+ <output-dir compare="Text">q13_customer_distribution</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q14_promotion_effect">
+ <output-dir compare="Text">q14_promotion_effect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q15_top_supplier">
+ <output-dir compare="Text">q15_top_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q16_parts_supplier_relationship">
+ <output-dir compare="Text">q16_parts_supplier_relationship</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q17_small_quantity_order_revenue">
+ <output-dir compare="Text">q17_small_quantity_order_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q17_large_gby_variant">
+ <output-dir compare="Text">q17_large_gby_variant</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q17_large_gby_variant_parameter">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Invalid query parameter compiler.groupmemory -- value has to be greater than or equal to</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q18_large_volume_customer">
+ <output-dir compare="Text">q18_large_volume_customer</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q19_discounted_revenue">
+ <output-dir compare="Text">q19_discounted_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q01_pricing_summary_report_nt">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q01_pricing_summary_report_2">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q01_pricing_summary_report_parallelism">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q01_pricing_summary_report_parameter">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Invalid query parameter compiler.sortmemory -- value has to be greater than or equal to</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q20_potential_part_promotion">
+ <output-dir compare="Text">q20_potential_part_promotion</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q21_suppliers_who_kept_orders_waiting">
+ <output-dir compare="Text">q21_suppliers_who_kept_orders_waiting</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q22_global_sales_opportunity">
+ <output-dir compare="Text">q22_global_sales_opportunity</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q02_minimum_cost_supplier">
+ <output-dir compare="Text">q02_minimum_cost_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q03_shipping_priority_nt">
+ <output-dir compare="Text">q03_shipping_priority_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q04_order_priority">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q05_local_supplier_volume">
+ <output-dir compare="Text">q05_local_supplier_volume</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q06_forecast_revenue_change">
+ <output-dir compare="Text">q06_forecast_revenue_change</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q07_volume_shipping">
+ <output-dir compare="Text">q07_volume_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q08_national_market_share">
+ <output-dir compare="Text">q08_national_market_share</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q09_product_type_profit_nt">
+ <output-dir compare="Text">q09_product_type_profit_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q09_product_type_profit_parameter">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Invalid query parameter compiler.joinmemory -- value has to be greater than or equal to</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue562">
+ <output-dir compare="Text">query-issue562</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue810">
+ <output-dir compare="Text">query-issue810</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue810-2">
+ <output-dir compare="Text">query-issue810-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="query-issue810-3">
+ <output-dir compare="Text">query-issue810-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tpch-with-index">
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="distinct_by">
+ <output-dir compare="Text">distinct_by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="nest_aggregate">
+ <output-dir compare="Text">nest_aggregate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="nest_aggregate2">
+ <output-dir compare="Text">nest_aggregate2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue785">
+ <output-dir compare="Text">query-issue785</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue785-2">
+ <output-dir compare="Text">query-issue785-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue786">
+ <output-dir compare="Text">query-issue786</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue601">
+ <output-dir compare="Text">query-issue601</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q10_returned_item">
+ <output-dir compare="Text">q10_returned_item</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q10_returned_item_int64">
+ <output-dir compare="Text">q10_returned_item_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q11_important_stock">
+ <output-dir compare="Text">q11_important_stock</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q12_shipping">
+ <output-dir compare="Text">q12_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q13_customer_distribution">
+ <output-dir compare="Text">q13_customer_distribution</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q14_promotion_effect">
+ <output-dir compare="Text">q14_promotion_effect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q15_top_supplier">
+ <output-dir compare="Text">q15_top_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q16_parts_supplier_relationship">
+ <output-dir compare="Text">q16_parts_supplier_relationship</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q17_small_quantity_order_revenue">
+ <output-dir compare="Text">q17_small_quantity_order_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q17_large_gby_variant">
+ <output-dir compare="Text">q17_large_gby_variant</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q18_large_volume_customer">
+ <output-dir compare="Text">q18_large_volume_customer</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q19_discounted_revenue">
+ <output-dir compare="Text">q19_discounted_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q01_pricing_summary_report_nt">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q20_potential_part_promotion">
+ <output-dir compare="Text">q20_potential_part_promotion</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q21_suppliers_who_kept_orders_waiting">
+ <output-dir compare="Text">q21_suppliers_who_kept_orders_waiting</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q22_global_sales_opportunity">
+ <output-dir compare="Text">q22_global_sales_opportunity</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q02_minimum_cost_supplier">
+ <output-dir compare="Text">q02_minimum_cost_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q03_shipping_priority_nt">
+ <output-dir compare="Text">q03_shipping_priority_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q04_order_priority">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q04_order_priority_with_nodegroup">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q05_local_supplier_volume">
+ <output-dir compare="Text">q05_local_supplier_volume</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q06_forecast_revenue_change">
+ <output-dir compare="Text">q06_forecast_revenue_change</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q07_volume_shipping">
+ <output-dir compare="Text">q07_volume_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q08_national_market_share">
+ <output-dir compare="Text">q08_national_market_share</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="q09_product_type_profit_nt">
+ <output-dir compare="Text">q09_product_type_profit_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue562">
+ <output-dir compare="Text">query-issue562</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue810">
+ <output-dir compare="Text">query-issue810</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue810-2">
+ <output-dir compare="Text">query-issue810-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue810-3">
+ <output-dir compare="Text">query-issue810-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue827">
+ <output-dir compare="Text">query-issue827</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-with-index">
+ <compilation-unit name="query-issue827-2">
+ <output-dir compare="Text">query-issue827-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tpch-sql-like">
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="query-issue638">
+ <output-dir compare="Text">query-issue638</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="query-issue785">
+ <output-dir compare="Text">query-issue785</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="query-issue785-2">
+ <output-dir compare="Text">query-issue785-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="query-issue786">
+ <output-dir compare="Text">query-issue786</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="query-issue601">
+ <output-dir compare="Text">query-issue601</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q10_returned_item">
+ <output-dir compare="Text">q10_returned_item</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q10_returned_item_int64">
+ <output-dir compare="Text">q10_returned_item_int64</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q11_important_stock">
+ <output-dir compare="Text">q11_important_stock</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q12_shipping">
+ <output-dir compare="Text">q12_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q13_customer_distribution">
+ <output-dir compare="Text">q13_customer_distribution</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q14_promotion_effect">
+ <output-dir compare="Text">q14_promotion_effect</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q15_top_supplier">
+ <output-dir compare="Text">q15_top_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q16_parts_supplier_relationship">
+ <output-dir compare="Text">q16_parts_supplier_relationship</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q17_small_quantity_order_revenue">
+ <output-dir compare="Text">q17_small_quantity_order_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q17_large_gby_variant">
+ <output-dir compare="Text">q17_large_gby_variant</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q18_large_volume_customer">
+ <output-dir compare="Text">q18_large_volume_customer</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q19_discounted_revenue">
+ <output-dir compare="Text">q19_discounted_revenue</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q01_pricing_summary_report_nt">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q20_potential_part_promotion">
+ <output-dir compare="Text">q20_potential_part_promotion</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q21_suppliers_who_kept_orders_waiting">
+ <output-dir compare="Text">q21_suppliers_who_kept_orders_waiting</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q22_global_sales_opportunity">
+ <output-dir compare="Text">q22_global_sales_opportunity</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q02_minimum_cost_supplier">
+ <output-dir compare="Text">q02_minimum_cost_supplier</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q03_shipping_priority_nt">
+ <output-dir compare="Text">q03_shipping_priority_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q04_order_priority">
+ <output-dir compare="Text">q04_order_priority</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q05_local_supplier_volume">
+ <output-dir compare="Text">q05_local_supplier_volume</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q06_forecast_revenue_change">
+ <output-dir compare="Text">q06_forecast_revenue_change</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q07_volume_shipping">
+ <output-dir compare="Text">q07_volume_shipping</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q08_national_market_share">
+ <output-dir compare="Text">q08_national_market_share</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-like">
+ <compilation-unit name="q09_product_type_profit_nt">
+ <output-dir compare="Text">q09_product_type_profit_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="writers">
+ <test-case FilePath="writers">
+ <compilation-unit name="print_01">
+ <output-dir compare="Text">print_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- TODO(madhusudancs): Enable this test when REST API supports serialized output support.
+ <test-case FilePath="writers">
+ <compilation-unit name="serialized_01">
+ <output-dir compare="Text">serialized_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ </test-group>
+ <test-group name="cross-dataverse">
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv01">
+ <output-dir compare="Text">cross-dv01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv02">
+ <output-dir compare="Text">cross-dv02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv03">
+ <output-dir compare="Text">cross-dv03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv04">
+ <output-dir compare="Text">cross-dv04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv07">
+ <output-dir compare="Text">cross-dv07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--NotImplementedException: No binary comparator factory implemented for type OBJECT.
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv08">
+ <output-dir compare="Text">cross-dv08</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv09">
+ <output-dir compare="Text">cross-dv09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv11">
+ <output-dir compare="Text">cross-dv11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv12">
+ <output-dir compare="Text">cross-dv12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv14">
+ <output-dir compare="Text">cross-dv14</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv15">
+ <output-dir compare="Text">cross-dv15</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--NotImplementedException: No binary comparator factory implemented for type OBJECT.
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv17">
+ <output-dir compare="Text">cross-dv17</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!--NotImplementedException: No binary comparator factory implemented for type OBJECT.
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv18">
+ <output-dir compare="Text">cross-dv18</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv19">
+ <output-dir compare="Text">cross-dv19</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="cross-dv20">
+ <output-dir compare="Text">cross-dv20</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="insert_across_dataverses">
+ <output-dir compare="Text">insert_across_dataverses</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="join_across_dataverses">
+ <output-dir compare="Text">join_across_dataverses</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="drop-dataverse">
+ <output-dir compare="Text">drop-dataverse</output-dir>
+ <expected-error>ASX1147: Cannot drop dataverse: type a.a being used by dataset b.b1</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: synonym a.s1 being used by function b.f1()</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="drop-type-used-elsewhere">
+ <output-dir compare="Text">drop-type-used-elsewhere</output-dir>
+ <expected-error>Cannot drop type a.a being used by dataset b.b1</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="drop-type-used-here-dataset">
+ <output-dir compare="Text">drop-type-used-here-dataset</output-dir>
+ <expected-error>Cannot drop type c.a being used by dataset c.a1</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="drop-type-used-here-type">
+ <output-dir compare="Text">drop-type-used-here-type</output-dir>
+ <expected-error>Cannot drop type c.a being used by type c.b</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="cross-dataverse">
+ <compilation-unit name="query-dataset-with-foreign-type">
+ <output-dir compare="Text">query-dataset-with-foreign-type</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="user-defined-functions">
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-1">
+ <output-dir compare="Text">bad-function-ddl-1</output-dir>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages</expected-error>
+ <expected-error>Cannot find dataset TweetMessages2 in dataverse experiments2 nor an alias with name TweetMessages2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-2">
+ <output-dir compare="Text">bad-function-ddl-2</output-dir>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments2 nor an alias with name TweetMessages</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-3">
+ <output-dir compare="Text">bad-function-ddl-3</output-dir>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-4">
+ <output-dir compare="Text">bad-function-ddl-4</output-dir>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experients nor an alias with name TweetMessages</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-5">
+ <output-dir compare="Text">bad-function-ddl-5</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature experiments.function_that_does_not_exist()</expected-error>
+ <expected-error>ASX1081: Cannot find function with signature experiments2.function_that_does_not_exist()</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-6">
+ <output-dir compare="Text">bad-function-ddl-6</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature experiments.function_that_does_not_exist(2)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-7">
+ <output-dir compare="Text">bad-function-ddl-7</output-dir>
+ <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-8">
+ <output-dir compare="Text">bad-function-ddl-8</output-dir>
+ <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-9">
+ <output-dir compare="Text">bad-function-ddl-9</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature experiments.function_that_does_not_exist()</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-10">
+ <output-dir compare="Text">bad-function-ddl-10</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature experiments.f0(2)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="bad-function-ddl-11">
+ <output-dir compare="Text">bad-function-ddl-11</output-dir>
+ <expected-error>ASX1001: Syntax error: Unexpected type declaration for parameter a in function myfn001</expected-error>
+ <expected-error>ASX1001: Syntax error: Unexpected return type declaration for function myfn002</expected-error>
+ <expected-error>ASX1001: Syntax error: Unexpected return type declaration for function myfn003</expected-error>
+ <expected-error>ASX1001: Syntax error: Unexpected IF NOT EXISTS (in line 29, at column 45)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="check-dependencies-1">
+ <output-dir compare="Text">check-dependencies-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="create-or-replace-function-1">
+ <output-dir compare="Text">create-or-replace-function-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-1">
+ <output-dir compare="Text">drop-dependency-1</output-dir>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f1(2) being used by function B.f0(2)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f2(...) being used by function B.f3(2)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f4(2) being used by function B.f5(...)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f6(...) being used by function B.f7(...)</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-2">
+ <output-dir compare="Text">drop-dependency-2</output-dir>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset (or view) C.TweetMessages being used by function B.f2(2)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset (or view) C.TweetMessages being used by function B.f3(...)</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-3">
+ <output-dir compare="Text">drop-dependency-3</output-dir>
+ <expected-error>Cannot drop function C.f1(2) being used by function B.f0(2)</expected-error>
+ <expected-error>Cannot drop function C.f3(...) being used by function B.f2(2)</expected-error>
+ <expected-error>Cannot drop function C.f5(2) being used by function B.f4(...)</expected-error>
+ <expected-error>Cannot drop function C.f7(...) being used by function B.f6(...)</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-4">
+ <output-dir compare="Text">drop-dependency-4</output-dir>
+ <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function B.f2(2)</expected-error>
+ <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function B.f2(...)</expected-error>
+ <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function B.f2(2)</expected-error>
+ <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function B.f2(...)</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-5">
+ <output-dir compare="Text">drop-dependency-5</output-dir>
+ <expected-error>Cannot drop function C.f1(2) being used by function C.f0(2)</expected-error>
+ <expected-error>Cannot drop function C.f1(2) being used by function C.f0(...)</expected-error>
+ <expected-error>Cannot drop function C.f1(...) being used by function C.f0(2)</expected-error>
+ <expected-error>Cannot drop function C.f1(...) being used by function C.f0(...)</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-6">
+ <output-dir compare="Text">drop-dependency-6</output-dir>
+ <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function C.f2(2)</expected-error>
+ <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function C.f2(...)</expected-error>
+ <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function C.f2(2)</expected-error>
+ <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function C.f2(...)</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-function-1">
+ <output-dir compare="Text">drop-function-1</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature experiments.my_sum(2)</expected-error>
+ <expected-error>ASX1081: Cannot find function with signature experiments.my_sum_va(2)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="single-line-definition">
+ <output-dir compare="Text">single-line-definition</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-ASTERIXDB-1298">
+ <output-dir compare="Text">query-ASTERIXDB-1298</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-ASTERIXDB-1652">
+ <output-dir compare="Text">query-ASTERIXDB-1652-2</output-dir>
+ <expected-error>ASX1063: Cannot find dataverse with name test</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-ASTERIXDB-1652-2">
+ <output-dir compare="Text">query-ASTERIXDB-1652-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-issue218-2">
+ <output-dir compare="Text">query-issue218-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-issue218">
+ <output-dir compare="Text">query-issue218</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-issue201">
+ <output-dir compare="Text">query-issue201</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-issue172">
+ <output-dir compare="Text">query-issue172</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-issue455">
+ <output-dir compare="Text">query-issue455</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature test.printName()</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-issue489">
+ <output-dir compare="Text">query-issue489</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf01">
+ <output-dir compare="Text">udf01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf02">
+ <output-dir compare="Text">udf02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf03">
+ <output-dir compare="Text">udf03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf04">
+ <output-dir compare="Text">udf04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf05">
+ <output-dir compare="Text">udf05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf06">
+ <output-dir compare="Text">udf06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf07">
+ <output-dir compare="Text">udf07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf08">
+ <output-dir compare="Text">udf08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf09">
+ <output-dir compare="Text">udf09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf10">
+ <output-dir compare="Text">udf10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf11">
+ <output-dir compare="Text">udf11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf12">
+ <output-dir compare="Text">udf12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf13">
+ <output-dir compare="Text">udf13</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf14">
+ <output-dir compare="Text">udf14</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf15">
+ <output-dir compare="Text">udf15</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf16">
+ <output-dir compare="Text">udf16</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf17">
+ <output-dir compare="Text">udf17</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf18">
+ <output-dir compare="Text">udf18</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf19">
+ <output-dir compare="Text">udf19</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf20">
+ <output-dir compare="Text">udf20</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf21">
+ <output-dir compare="Text">udf21</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf22">
+ <output-dir compare="Text">udf22</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf23">
+ <output-dir compare="Text">udf23</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf24">
+ <output-dir compare="Text">udf24</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf25">
+ <output-dir compare="Text">udf25</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf27">
+ <output-dir compare="Text">udf27</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf28">
+ <output-dir compare="Text">udf28</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf29">
+ <output-dir compare="Text">udf29</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- Need to verify the expected exception -->
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf30">
+ <output-dir compare="Text">udf30</output-dir>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier y (in line 30, at column 8)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf31">
+ <output-dir compare="Text">udf31</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf32_metadata">
+ <output-dir compare="Text">udf32_metadata</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf33_overloading">
+ <output-dir compare="Text">udf33_overloading</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf34">
+ <output-dir compare="Text">udf34</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf35_varargs_misc">
+ <output-dir compare="Text">udf35_varargs_misc</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf36_in_with">
+ <output-dir compare="Text">udf36_in_with</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf37_recursion">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 24, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 25, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 26, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 27, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 28, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 30, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 31, at column 1)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf38_no_recursion">
+ <output-dir compare="Text">udf38_no_recursion</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="udf39_illegal_call">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1150: Illegal use of function test.f1a(1) (in line 32, at column 26)</expected-error>
+ <expected-error>ASX1150: Illegal use of function test.f2a(...) (in line 29, at column 28)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="f01">
+ <output-dir compare="Text">f01</output-dir>
+ <expected-error>ASX1081: Cannot find function with signature test.tinyint()</expected-error>
+ </compilation-unit>
+ </test-case>
+ <!-- This test case is not valid anymore since we do not required "IMPORT_PRIVATE_FUNCTIONS" flag anymore -->
+ <!-- <test-case FilePath="user-defined-functions">
+ <compilation-unit name="invoke-private-function">
+ <output-dir compare="Text">invoke-private-function</output-dir>
+ </compilation-unit>
+ </test-case>-->
+ <!--
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-ASTERIXDB-1308-1">
+ <output-dir compare="Text">query-ASTERIXDB-1308-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <!-- <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-ASTERIXDB-1308-2">
+ <output-dir compare="Text">query-ASTERIXDB-1308-2</output-dir>
+ </compilation-unit>
+ </test-case> -->
+ <!-- This test case is not valid anymore since we do not required "IMPORT_PRIVATE_FUNCTIONS" flag anymore -->
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-ASTERIXDB-1317">
+ <output-dir compare="Text">query-ASTERIXDB-1317</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="view">
+ <test-case FilePath="view">
+ <compilation-unit name="create-view-1">
+ <output-dir compare="Text">create-view-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="create-view-2-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1063: Cannot find dataverse with name test (in line 24, at column 1)</expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 25 >>create view test.v1 as select * from range(1,2) r order by;<< Encountered ";" at column 59]]></expected-error>
+ <expected-error>ASX1081: Cannot find function with signature test.undefined_range(2) (in line 25, at column 38)</expected-error>
+ <expected-error>ASX1160: A view with this name test.v1 already exists (in line 26, at column 1)</expected-error>
+ <expected-error>ASX1072: A dataset with name ds1 already exists in dataverse test (in line 30, at column 1)</expected-error>
+ <expected-error>ASX1072: A dataset with name ds2 already exists in dataverse test (in line 30, at column 1)</expected-error>
+ <expected-error>ASX1160: A view with this name test.ds1 already exists (in line 27, at column 1)</expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: Unexpected IF NOT EXISTS (in line 25, at column 1)]]></expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 31, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 32, at column 1)</expected-error>
+ <expected-error>ASX1149: Illegal function or view recursion (in line 33, at column 1)</expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 25 >>create view test.v1 primary key (r) not enforced as<< Encountered "primary" at column 21]]></expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 28 >>create view test.v2 foreign key (r) references v1 not enforced as<< Encountered <IDENTIFIER> "foreign" at column 21]]></expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="create-view-3-typed">
+ <output-dir compare="Text">create-view-3-typed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view" check-warnings="true">
+ <compilation-unit name="create-view-4-typed-warn">
+ <output-dir compare="Text">create-view-4-typed-warn</output-dir>
+ <expected-warn>ASX0006: Invalid format for tinyint in a (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for smallint in b (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for integer in c (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for bigint in d (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for float in e (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for double in f (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for datetime in g (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for date in h (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for time in j (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for duration in k (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for yearmonthduration in m (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for daytimeduration in n (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: boolean-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int8-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int16-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int32-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: int64-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: float-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: double-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: datetime-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: date-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: time-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: duration-default-null() cannot process input type date (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: year-month-duration-default-null() cannot process input type date (in line 30, at column 6)</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: day-time-duration-default-null() cannot process input type date (in line 30, at column 6)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view" check-warnings="true">
+ <compilation-unit name="create-view-5-typed-warn">
+ <output-dir compare="Text">create-view-5-typed-warn</output-dir>
+ <expected-warn>ASX0006: Invalid format for datetime in a (in line 27, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for date in b (in line 27, at column 6)</expected-warn>
+ <expected-warn>ASX0006: Invalid format for time in c (in line 27, at column 6)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="create-view-6-typed-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1082: Cannot find datatype with name unknown_dv.t1</expected-error>
+ <expected-error>ASX1082: Cannot find datatype with name test.t1_unknown</expected-error>
+ <expected-error>ASX1079: Compilation error: view type cannot have open fields (in line 29, at column 1)</expected-error>
+ <expected-error>ASX1004: Unsupported type: view cannot process input type t1_a (in line 30, at column 1)</expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 25 >>create view test.v1(r bigint, a [bigint]) default null as<< Encountered "[" at column 33]]></expected-error>
+ <expected-error>ASX1092: Parameter date_illegal_property_name cannot be set (in line 25, at column 1)</expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 25 >>create view test.v1(r bigint) as<< Encountered "as" at column 31]]></expected-error>
+ <expected-error><![CDATA[ASX1014: Field 'unknown_field' is not found (in line 25, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1014: Field 'unknown_field_2' is not found (in line 25, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 28 >> as select r from range(1,2) r;<< Encountered "as" at column 3]]></expected-error>
+ <expected-error><![CDATA[ASX0013: Duplicate field name 'r' (in line 25, at column 20)]]></expected-error>
+ <expected-error><![CDATA[ASX1167: Cannot change primary key of view test1.employee_v1 (in line 38, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1162: Invalid primary key definition (in line 25, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1162: Invalid primary key definition (in line 26, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1001: Syntax error: In line 36 >> as employee;<< Encountered "as" at column 3]]></expected-error>
+ <expected-error><![CDATA[ASX1165: Invalid foreign key definition: view test1.employee_v1 does not have a primary key (in line 32, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1166: Invalid foreign key definition: foreign key does not match primary key of view test1.employee_v1 (in line 34, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1166: Invalid foreign key definition: foreign key does not match primary key of view test1.employee_v2 (in line 34, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1164: Invalid foreign key definition (in line 34, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1063: Cannot find dataverse with name test3 (in line 42, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1159: Cannot find view with name test1.employee_v3 (in line 42, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1159: Cannot find view with name test1.employee (in line 43, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1164: Invalid foreign key definition (in line 43, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1164: Invalid foreign key definition (in line 39, at column 1)]]></expected-error>
+ <expected-error><![CDATA[ASX1166: Invalid foreign key definition: foreign key does not match primary key of view test1.employee_v1 (in line 39, at column 1)]]></expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="create-view-7-foreign-key">
+ <output-dir compare="Text">create-view-7-foreign-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="drop-dataverse-1">
+ <output-dir compare="Text">drop-dataverse-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="drop-dataverse-2-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset (or view) test2.v2 being used by view test1.v1</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset (or view) test2.v2 being used by function test1.f1()</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset (or view) test2.ds2 being used by view test1.v1</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function test2.f2() being used by view test1.v1</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: synonym test2.s3 being used by view test1.v1</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: type test2.t1 being used by dataset test1.v1</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset (or view) test2.employee_v2 being used by view test1.employee_v1</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="drop-view-1">
+ <output-dir compare="Text">drop-view-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="drop-view-2-negative">
+ <output-dir compare="Text">drop-view-2-negative</output-dir>
+ <expected-error>ASX1063: Cannot find dataverse with name test (in line 24, at column 1)</expected-error>
+ <expected-error>ASX1159: Cannot find view with name test.v1 (in line 25, at column 1)</expected-error>
+ <expected-error>ASX1050: Cannot find dataset with name v1 in dataverse test (in line 27, at column 1)</expected-error>
+ <expected-error>ASX1159: Cannot find view with name test.ds1 (in line 30, at column 1)</expected-error>
+ <expected-error>ASX1148: Cannot drop dataset test2.ds2 being used by view test1.v1</expected-error>
+ <expected-error>ASX1148: Cannot drop function test2.f2() being used by view test1.v1</expected-error>
+ <expected-error>ASX1148: Cannot drop synonym test2.s2 being used by view test1.v1</expected-error>
+ <expected-error>ASX1148: Cannot drop view test2.v2 being used by view test1.v1</expected-error>
+ <expected-error>ASX1148: Cannot drop view test2.v2 being used by function test1.f1()</expected-error>
+ <expected-error>ASX1148: Cannot drop type test2.t1 being used by dataset test1.v1</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="view-1">
+ <output-dir compare="Text">view-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="view">
+ <compilation-unit name="view-2-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1050: Cannot find dataset with name v1 in dataverse test1 (in line 24, at column 17)</expected-error>
+ <expected-error>ASX1050: Cannot find dataset with name v2 in dataverse test1 (in line 24, at column 17)</expected-error>
+ <expected-error>ASX1050: Cannot find dataset with name v3 in dataverse test1 (in line 24, at column 1)</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="load">
+ <test-case FilePath="load">
+ <compilation-unit name="load_non-empty_index">
+ <output-dir compare="Text">load_non-empty_index</output-dir>
+ <expected-error>HYR0034: Cannot load an index that is not empty</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_01">
+ <output-dir compare="Text">csv_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_02">
+ <output-dir compare="Text">csv_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_03">
+ <output-dir compare="Text">csv_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_04">
+ <output-dir compare="Text">csv_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_05">
+ <output-dir compare="Text">csv_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_06">
+ <output-dir compare="Text">csv_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_07">
+ <output-dir compare="Text">csv_07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_08_header_cr">
+ <output-dir compare="Text">csv_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_08_header_lf">
+ <output-dir compare="Text">csv_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="csv_08_header_crlf">
+ <output-dir compare="Text">csv_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="issue14_query">
+ <output-dir compare="Text">issue14_query</output-dir>
+ <expected-error>Unspecified parameter: format</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="issue315_query">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Invalid path</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="issue289_query">
+ <output-dir compare="Text">issue289_query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="issue650_query">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Cannot find dataset with name Users in dataverse fuzzyjoin</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="type_promotion_0">
+ <output-dir compare="Text">type_promotion_0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="escapes01">
+ <output-dir compare="Text">escapes01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="escapes02">
+ <output-dir compare="Text">escapes02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="escapes-err-1"><!-- Exception is never thrown!!!. needs to be investigated -->
+ <output-dir compare="Text">none</output-dir>
+ <!-- <expected-error>org.apache.hyracks.api.exceptions.HyracksException</expected-error> -->
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="query-issue244">
+ <output-dir compare="Text">query-issue244</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="duplicate-key-error">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Loading duplicate keys into the primary storage</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="issue610_adm_token_end_collection">
+ <output-dir compare="Text">issue610_adm_token_end_collection</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="adm_binary">
+ <output-dir compare="Text">adm_binary</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="dataset-with-meta">
+ <output-dir compare="Text">dataset-with-meta</output-dir>
+ <expected-error>ASX1079: Compilation error: DatasetWithMeta: load dataset is not supported on datasets with meta records (in line 27, at column 1)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="escapes-err-1">
+ <output-dir compare="Text">escapes-err-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="escapes01">
+ <output-dir compare="Text">escapes01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="escapes02">
+ <output-dir compare="Text">escapes02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="file-not-found">
+ <output-dir compare="Text">file-not-found</output-dir>
+ <expected-error>ASX3077: bla: path not found</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="type_promotion_0">
+ <output-dir compare="Text">type_promotion_0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="load">
+ <compilation-unit name="utf8">
+ <output-dir compare="Text">utf8</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="hints">
+ <test-case FilePath="hints">
+ <compilation-unit name="issue_251_dataset_hint_5">
+ <output-dir compare="Text">issue_251_dataset_hint_5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="hints">
+ <compilation-unit name="issue_251_dataset_hint_7">
+ <output-dir compare="Text">issue_251_dataset_hint_7</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="function">
+ <test-case FilePath="function">
+ <compilation-unit name="issue-2394">
+ <output-dir compare="Text">issue-2394</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="function">
+ <compilation-unit name="drop_if_exists">
+ <output-dir compare="Text">drop_if_exists</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="feeds">
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_07">
+ <output-dir compare="Text">feeds_07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_08">
+ <output-dir compare="Text">feeds_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_09">
+ <output-dir compare="Text">feeds_09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="create-policy-from-file">
+ <output-dir compare="Text">create-policy-from-file</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_01">
+ <output-dir compare="Text">feeds_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_02">
+ <output-dir compare="Text">feeds_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_03">
+ <output-dir compare="Text">feeds_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_10">
+ <output-dir compare="Text">feeds_10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_11">
+ <output-dir compare="Text">feeds_11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_12">
+ <output-dir compare="Text">feeds_12</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_13">
+ <output-dir compare="Text">feeds_13</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="issue_230_feeds">
+ <output-dir compare="Text">issue_230_feeds</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="insert-feed">
+ <output-dir compare="Text">insert-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="insert-feed-with-pk-index">
+ <output-dir compare="Text">insert-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="connect-feed-with-function">
+ <output-dir compare="Text">connect-feed-with-function</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-filter-on-meta-dataset">
+ <output-dir compare="Text">change-feed-filter-on-meta-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-index">
+ <output-dir compare="Text">change-feed-with-meta-pk-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-with-mixed-index">
+ <output-dir compare="Text">change-feed-with-meta-with-mixed-index</output-dir>
+ <expected-error>Compilation error: Cannot create index on meta fields (in line 61, at column 35)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest">
+ <output-dir compare="Text">change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-in-meta-open-index-with-missing">
+ <output-dir compare="Text">change-feed-with-meta-pk-in-meta-open-index-with-missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-open-index-in-meta">
+ <output-dir compare="Text">change-feed-with-meta-open-index-in-meta</output-dir>
+ <expected-error>Compilation error: Cannot create index on meta fields (in line 60, at column 34)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-in-meta-open-index-in-value">
+ <output-dir compare="Text">change-feed-with-meta-pk-in-meta-open-index-in-value</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-in-meta-index-after-ingest">
+ <output-dir compare="Text">change-feed-with-meta-pk-in-meta-index-after-ingest</output-dir>
+ <expected-error>Compilation error: Cannot create index on meta fields (in line 27, at column 37)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-in-meta-index-in-meta">
+ <output-dir compare="Text">change-feed-with-meta-pk-in-meta-index-in-meta</output-dir>
+ <expected-error>Compilation error: Cannot create index on meta fields (in line 60, at column 37)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-csv">
+ <output-dir compare="Text">change-feed-with-meta-csv</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-in-meta">
+ <output-dir compare="Text">change-feed-with-meta-pk-in-meta</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed">
+ <output-dir compare="Text">change-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="drop-nonexistent-feed">
+ <output-dir compare="Text">drop-nonexistent-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="twitter-feed">
+ <output-dir compare="Text">twitter-feed</output-dir>
+ <expected-error>Twitter4J library not found!</expected-error>
+ <expected-error>Unknown source feed: TwitterFeed</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="revised-tweet-parser">
+ <output-dir compare="Text">revised-tweet-parser</output-dir>
+ <expected-error>Twitter4J library not found!</expected-error>
+ <expected-error>Unknown source feed: TwitterFeed</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="connect-feed">
+ <output-dir compare="Text">connect-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-meta-pk-in-meta">
+ <output-dir compare="Text">change-feed-with-meta-pk-in-meta</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feed-with-filtered-dataset">
+ <output-dir compare="Text">feed-with-filtered-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed">
+ <output-dir compare="Text">change-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="drop-dataverse-with-disconnected-feed">
+ <output-dir compare="Text">drop-dataverse-with-disconnected-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feed-push-socket">
+ <output-dir compare="Text">feed-push-socket</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="insert-feed">
+ <output-dir compare="Text">insert-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="start-feed">
+ <output-dir compare="Text">start-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="start-started-feed">
+ <output-dir compare="Text">start-started-feed</output-dir>
+ <expected-error>experiments.TweetFeed(Feed) is already started</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="stop-stopped-feed">
+ <output-dir compare="Text">stop-stopped-feed</output-dir>
+ <expected-error>new_experiments.TweetFeed(Feed) cannot be stopped because its state is STOPPED</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="push-socket-with-auuid">
+ <output-dir compare="Text">push-socket-with-auuid</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="disconnect-live-feed">
+ <output-dir compare="Text">disconnect-live-feed</output-dir>
+ <expected-error>This operation cannot be done when Feed</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="connect-live-feed">
+ <output-dir compare="Text">connect-live-feed</output-dir>
+ <expected-error>This operation cannot be done when Feed</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="record-reader-with-malformed-input-stream">
+ <output-dir compare="Text">record-reader-with-malformed-input-stream</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feed-with-undefined-function">
+ <output-dir compare="Text">feed-with-undefined-function</output-dir>
+ <expected-error>Cannot find function</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="drop-function-used-by-feed">
+ <output-dir compare="Text">drop-function-used-by-feed</output-dir>
+ <expected-error>ASX1148: Cannot drop function experiments.test_func0(1) being used by feed connection experiments.UserFeed</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="drop-function-no-longer-used-by-feed">
+ <output-dir compare="Text">drop-function-used-by-feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="drop-dataverse-with-function-used-by-feed">
+ <output-dir compare="Text">drop-dataverse-with-function-used-by-feed</output-dir>
+ <expected-error>ASX1147: Cannot drop dataverse: function fundv.test_func0(1) being used by feed connection feeddv.UserFeed</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="http_feed">
+ <output-dir compare="Text">http_feed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="http_feed_json">
+ <output-dir compare="Text">http_feed_json</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="change-feed-with-where-on-meta">
+ <output-dir compare="Text">change-feed-with-where-on-meta</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="meta">
+ <test-case FilePath="meta">
+ <compilation-unit name="meta_in_with_clause">
+ <output-dir compare="Text">meta_in_with_clause</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="resolving_pk_with_meta">
+ <output-dir compare="Text">resolving_pk_with_meta</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="meta_after_gby">
+ <output-dir compare="Text">meta_after_gby</output-dir>
+ <expected-error>Compilation error: Inappropriate use of function 'meta'. For example, after GROUP BY (in line 29, at column 21)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="query_dataset_with_meta-1">
+ <output-dir compare="Text">query_dataset_with_meta-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="query_dataset_with_meta-2">
+ <output-dir compare="Text">query_dataset_with_meta-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="query_dataset_with_meta_failure">
+ <output-dir compare="Text">query_dataset_with_meta_failure</output-dir>
+ <expected-error>ASX1079: Compilation error: Cannot resolve ambiguous meta function call. There are more than one dataset choice (in line 24, at column 7)</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="big-object">
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_sort">
+ <output-dir compare="Text">big_object_sort</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_groupby">
+ <output-dir compare="Text">big_object_groupby</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_groupby-2">
+ <output-dir compare="Text">big_object_groupby-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_join">
+ <output-dir compare="Text">big_object_join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_join_low_memory_err">
+ <output-dir compare="Text">big_object_join</output-dir>
+ <expected-error>HYR0123: Insufficient memory is provided for the join operators, please increase the join memory budget.</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_load_20M">
+ <output-dir compare="Text">big_object_load_20M</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_bulkload">
+ <output-dir compare="Text">big_object_bulkload</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_feed_20M">
+ <output-dir compare="Text">big_object_feed_20M</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_insert">
+ <output-dir compare="Text">big_object_insert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_load_only_20M">
+ <output-dir compare="Text">big_object_load_only_20M</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="leftouterjoin">
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="loj-01-core">
+ <output-dir compare="Text">loj-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="loj-01-sugar">
+ <output-dir compare="Text">loj-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="loj-02-push-select">
+ <output-dir compare="Text">loj-02-push-select</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="loj-03-no-listify">
+ <output-dir compare="Text">loj-03-no-listify</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="query_issue658">
+ <output-dir compare="Text">query_issue658</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="query_issue285">
+ <output-dir compare="Text">query_issue285</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="query_issue285-2">
+ <output-dir compare="Text">query_issue285-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="query_issue849">
+ <output-dir compare="Text">query_issue849</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="query_issue849-2">
+ <output-dir compare="Text">query_issue849-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="empty-dataset">
+ <output-dir compare="Text">empty-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="query-ASTERIXDB-769">
+ <output-dir compare="Text">query-ASTERIXDB-769</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="query-ASTERIXDB-2857">
+ <output-dir compare="Text">query-ASTERIXDB-2857</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
+ <compilation-unit name="right_branch_opt_1">
+ <output-dir compare="Text">right_branch_opt_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="index-leftouterjoin">
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-join-btree-sidx3-idxonly">
+ <output-dir compare="Text">probe-pidx-join-btree-sidx3-idxonly</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-idxonly-join-btree-pidx1">
+ <output-dir compare="Text">probe-sidx-btree-idxonly-join-btree-pidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-idxonly-join-btree-sidx1-idxonly">
+ <output-dir compare="Text">probe-sidx-btree-idxonly-join-btree-sidx1-idxonly</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-non-idxonly-join-btree-pidx1">
+ <output-dir compare="Text">probe-sidx-btree-non-idxonly-join-btree-pidx1</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <!--test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-sidx-btree-non-idxonly-join-btree-sidx1-idxonly">
+ <output-dir compare="Text">probe-sidx-btree-non-idxonly-join-btree-sidx1-idxonly</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-pidx1">
+ <output-dir compare="Text">probe-pidx-with-join-btree-pidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="distinct">
+ <test-case FilePath="distinct">
+ <compilation-unit name="array">
+ <output-dir compare="Text">array</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="distinct">
+ <compilation-unit name="record">
+ <output-dir compare="Text">record</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="distinct">
+ <compilation-unit name="query-issue443">
+ <output-dir compare="Text">query-issue443</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="distinct">
+ <compilation-unit name="query-issue443-2">
+ <output-dir compare="Text">query-issue443-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="rightouterjoin">
+ <test-case FilePath="rightouterjoin">
+ <compilation-unit name="roj-01-core">
+ <output-dir compare="Text">roj-01-core</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="rightouterjoin">
+ <compilation-unit name="roj-02-core">
+ <output-dir compare="Text">roj-02-core</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="rightouterjoin">
+ <compilation-unit name="roj-03-negative">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1130: Illegal use of RIGHT OUTER JOIN</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="tinysocial">
+ <test-case FilePath="tinysocial">
+ <compilation-unit name="tinysocial-suite">
+ <output-dir compare="Text">tinysocial-suite</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tinysocial">
+ <compilation-unit name="tinysocial-suite-open">
+ <output-dir compare="Text">tinysocial-suite-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="types">
+ <test-case FilePath="types">
+ <compilation-unit name="any-object">
+ <output-dir compare="Text">any-object</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="gettype">
+ <output-dir compare="Text">gettype</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isarray">
+ <output-dir compare="Text">isarray</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isatomic">
+ <output-dir compare="Text">isatomic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isboolean">
+ <output-dir compare="Text">isboolean</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isnumber">
+ <output-dir compare="Text">isnumber</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isobject">
+ <output-dir compare="Text">isobject</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isstring">
+ <output-dir compare="Text">isstring</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isbinary">
+ <output-dir compare="Text">isbinary</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="iscircle">
+ <output-dir compare="Text">iscircle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isdate">
+ <output-dir compare="Text">isdate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isdatetime">
+ <output-dir compare="Text">isdatetime</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isduration">
+ <output-dir compare="Text">isduration</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isinterval">
+ <output-dir compare="Text">isinterval</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isline">
+ <output-dir compare="Text">isline</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="ismultiset">
+ <output-dir compare="Text">ismultiset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="ispoint">
+ <output-dir compare="Text">ispoint</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="ispolygon">
+ <output-dir compare="Text">ispolygon</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isrectangle">
+ <output-dir compare="Text">isrectangle</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isspatial">
+ <output-dir compare="Text">isspatial</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="istemporal">
+ <output-dir compare="Text">istemporal</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="istime">
+ <output-dir compare="Text">istime</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="isuuid">
+ <output-dir compare="Text">isuuid</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="is_all_types">
+ <output-dir compare="Text">is_all_types</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="record01">
+ <output-dir compare="Text">record01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="type_promotion_0">
+ <output-dir compare="Text">type_promotion_0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="type_promotion_1">
+ <output-dir compare="Text">type_promotion_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="opentype_orderby_01">
+ <output-dir compare="Text">opentype_orderby_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_closedtype_field_01">
+ <output-dir compare="Text">promotion_closedtype_field_vs_closedtype_field_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_closedtype_field_02">
+ <output-dir compare="Text">promotion_closedtype_field_vs_closedtype_field_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_closedtype_field_03">
+ <output-dir compare="Text">promotion_closedtype_field_vs_closedtype_field_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_closedtype_field_04">
+ <output-dir compare="Text">promotion_closedtype_field_vs_closedtype_field_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_closedtype_field_05">
+ <output-dir compare="Text">promotion_closedtype_field_vs_closedtype_field_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_01">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_02">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_03">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_04">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_05">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_06">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_07">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_08">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_09">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_10">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_10</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_constant_11">
+ <output-dir compare="Text">promotion_closedtype_field_vs_constant_11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_opentype_field_01">
+ <output-dir compare="Text">promotion_closedtype_field_vs_opentype_field_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_opentype_field_02">
+ <output-dir compare="Text">promotion_closedtype_field_vs_opentype_field_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_opentype_field_03">
+ <output-dir compare="Text">promotion_closedtype_field_vs_opentype_field_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_closedtype_field_vs_opentype_field_04">
+ <output-dir compare="Text">promotion_closedtype_field_vs_opentype_field_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_01">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_02">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_03">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_04">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_05">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_05</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_06">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_07">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_07</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_constant_08">
+ <output-dir compare="Text">promotion_opentype_field_vs_constant_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_opentype_field_01">
+ <output-dir compare="Text">promotion_opentype_field_vs_opentype_field_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="promotion_opentype_field_vs_opentype_field_02">
+ <output-dir compare="Text">promotion_opentype_field_vs_opentype_field_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_array">
+ <output-dir compare="Text">to_array</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_atomic">
+ <output-dir compare="Text">to_atomic</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_boolean_01">
+ <output-dir compare="Text">to_boolean_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types" check-warnings="true">
+ <compilation-unit name="to_boolean_02">
+ <output-dir compare="Text">to_boolean_02</output-dir>
+ <expected-warn>ASX0004: Unsupported type: to-boolean() cannot process input type date (in line 24, at column 8)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_bigint_01">
+ <output-dir compare="Text">to_bigint_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types" check-warnings="true">
+ <compilation-unit name="to_bigint_02">
+ <output-dir compare="Text">to_bigint_02</output-dir>
+ <expected-warn>ASX0004: Unsupported type: to-bigint() cannot process input type date (in line 24, at column 7)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_double_01">
+ <output-dir compare="Text">to_double_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types" check-warnings="true">
+ <compilation-unit name="to_double_02">
+ <output-dir compare="Text">to_double_02</output-dir>
+ <expected-warn>ASX0004: Unsupported type: to-double() cannot process input type date (in line 24, at column 7)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_number_01">
+ <output-dir compare="Text">to_number_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_number_02">
+ <output-dir compare="Text">to_number_02</output-dir>
+ <expected-error>ASX0002: Type mismatch</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_object">
+ <output-dir compare="Text">to_object</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_string_01">
+ <output-dir compare="Text">to_string_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="to_string_02">
+ <output-dir compare="Text">to_string_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="domain_boundaries">
+ <output-dir compare="Text">domain_boundaries</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="domain_boundaries_error">
+ <output-dir compare="Text">domain_boundaries_error</output-dir>
+ <expected-error>ASX0001: Field type null cannot be promoted to type tinyint</expected-error>
+ <expected-error>ASX0001: Field type null cannot be promoted to type smallint</expected-error>
+ <expected-error>ASX0001: Field type null cannot be promoted to type integer</expected-error>
+ <expected-error>ASX0001: Field type null cannot be promoted to type bigint</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="types">
+ <compilation-unit name="query-ASTERIXDB-2950">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>ASX1002: Type mismatch: function get-item expects its 1st input parameter to be of type multiset or array, but the actual input type is bigint (in line 24, at column 21)</expected-error>
+ <expected-error>ASX1002: Type mismatch: function get-item expects its 1st input parameter to be of type multiset or array, but the actual input type is bigint (in line 25, at column 31)</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="cleanjson">
+ <test-case FilePath="json">
+ <compilation-unit name="issue-ASTERIXDB-1165">
+ <output-dir compare="Clean-JSON">issue-ASTERIXDB-1165</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json">
+ <compilation-unit name="int01">
+ <output-dir compare="Clean-JSON">int01-cleanjson</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="materialization">
+ <test-case FilePath="materialization">
+ <compilation-unit name="assign-reuse">
+ <output-dir compare="Text">assign-reuse</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="filters">
+ <test-case FilePath="filters">
+ <compilation-unit name="equality-predicate">
+ <output-dir compare="Text">equality-predicate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="filter-auto-key">
+ <output-dir compare="Text">filter-auto-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load">
+ <output-dir compare="Text">load</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-btree">
+ <output-dir compare="Text">load-with-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-btree-index-only">
+ <output-dir compare="Text">load-with-secondary-btree-index-only</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-inverted-ngram">
+ <output-dir compare="Text">load-with-secondary-inverted-ngram</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-inverted-word">
+ <output-dir compare="Text">load-with-secondary-inverted-word</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="load-with-secondary-rtree">
+ <output-dir compare="Text">load-with-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert">
+ <output-dir compare="Text">insert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-secondary-btree">
+ <output-dir compare="Text">insert-with-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-btree">
+ <output-dir compare="Text">insert-with-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-secondary-inverted-ngram">
+ <output-dir compare="Text">insert-with-secondary-inverted-ngram</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-inverted-ngram">
+ <output-dir compare="Text">insert-with-secondary-inverted-ngram</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-secondary-inverted-word">
+ <output-dir compare="Text">insert-with-secondary-inverted-word</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-inverted-word">
+ <output-dir compare="Text">insert-with-secondary-inverted-word</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-secondary-rtree">
+ <output-dir compare="Text">insert-with-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-rtree">
+ <output-dir compare="Text">insert-with-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="nested-filter-equality-predicate">
+ <output-dir compare="Text">nested-filter-equality-predicate</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="upsert">
+ <output-dir compare="Text">upsert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="delete">
+ <output-dir compare="Text">delete</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="json">
+ <test-case FilePath="json">
+ <compilation-unit name="int01">
+ <output-dir compare="Lossless-JSON">int01-losslessjson</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="csv">
+ <test-case FilePath="csv">
+ <compilation-unit name="basic-types">
+ <output-dir compare="CSV">basic-types</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="csv">
+ <compilation-unit name="basic-types">
+ <output-dir compare="CSV_Header">basic-types-header</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="csv-tsv-parser">
+ <test-case FilePath="csv-tsv-parser">
+ <compilation-unit name="csv-parser-001">
+ <output-dir compare="Text">csv-parser-001</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="csv-tsv-parser">
+ <compilation-unit name="tsv-parser-001">
+ <output-dir compare="Text">tsv-parser-001</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="binary">
+ <test-case FilePath="binary">
+ <compilation-unit name="parse">
+ <output-dir compare="Text">parse</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="print">
+ <output-dir compare="Text">print</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="concat">
+ <output-dir compare="Text">concat</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="concat2">
+ <output-dir compare="Text">concat2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="subbinary">
+ <output-dir compare="Text">subbinary</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="find">
+ <output-dir compare="Text">find</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="insert">
+ <output-dir compare="Text">insert</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="equal_join">
+ <output-dir compare="Text">equal_join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="index_join">
+ <output-dir compare="Text">index_join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="binary">
+ <compilation-unit name="length">
+ <output-dir compare="Text">length</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="unnest">
+ <test-case FilePath="unnest">
+ <compilation-unit name="ASTERIXDB-2750_unnest_join">
+ <output-dir compare="Text">ASTERIXDB-2750_unnest_join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="unnest">
+ <compilation-unit name="ASTERIXDB-2844_unnest_syntax">
+ <output-dir compare="Text">ASTERIXDB-2844_unnest_syntax</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="unnest">
+ <compilation-unit name="left-outer-unnest">
+ <output-dir compare="Text">left-outer-unnest</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="unnest">
+ <compilation-unit name="left-outer-unnest-with-pos">
+ <output-dir compare="Text">left-outer-unnest-with-pos</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="union">
+ <test-case FilePath="union">
+ <compilation-unit name="union">
+ <output-dir compare="Text">union</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_heterogeneous_scalar">
+ <output-dir compare="Text">union_heterogeneous_scalar</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_negative">
+ <output-dir compare="Text">union</output-dir>
+ <expected-error>Cannot find dataset t in dataverse TinySocial nor an alias with name t</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_negative_3">
+ <output-dir compare="Text">union</output-dir>
+ <expected-error>Operation UNION with set semantics is not supported.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_nested">
+ <output-dir compare="Text">union_nested</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_opt_1">
+ <output-dir compare="Text">union_opt_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_orderby">
+ <output-dir compare="Text">union_orderby</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_orderby_2">
+ <output-dir compare="Text">union_orderby_2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_orderby_3">
+ <output-dir compare="Text">union_orderby_3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_orderby_4">
+ <output-dir compare="Text">union_orderby_3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="union_orderby_5">
+ <output-dir compare="Text">union_orderby_5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="query-ASTERIXDB-1354-2">
+ <output-dir compare="Text">query-ASTERIXDB-1354-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="query-ASTERIXDB-1629">
+ <output-dir compare="Text">query-ASTERIXDB-1629</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="query-ASTERIXDB-1047">
+ <output-dir compare="Text">query-ASTERIXDB-1047</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="query-ASTERIXDB-1205-2">
+ <output-dir compare="Text">query-ASTERIXDB-1205-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="query-ASTERIXDB-1205-3">
+ <output-dir compare="Text">query-ASTERIXDB-1205-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="query-ASTERIXDB-1205">
+ <output-dir compare="Text">query-ASTERIXDB-1205</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="union">
+ <compilation-unit name="query-ASTERIXDB-1354">
+ <output-dir compare="Text">query-ASTERIXDB-1354</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="upsert">
+ <test-case FilePath="upsert">
+ <compilation-unit name="filtered-dataset">
+ <output-dir compare="Text">filtered-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="issue1587-foreignDataType">
+ <output-dir compare="Text">issue1587-foreignDataType</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="nested-index">
+ <output-dir compare="Text">nested-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-secondary-rtree">
+ <output-dir compare="Text">primary-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-rtree">
+ <output-dir compare="Text">primary-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="upsert-with-self-read">
+ <output-dir compare="Text">upsert-with-self-read</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="nullable-index">
+ <output-dir compare="Text">nullable-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="open-index">
+ <output-dir compare="Text">open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-index">
+ <output-dir compare="Text">primary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-secondary-btree">
+ <output-dir compare="Text">primary-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-btree">
+ <output-dir compare="Text">primary-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-secondary-inverted">
+ <output-dir compare="Text">primary-secondary-inverted</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-inverted">
+ <output-dir compare="Text">primary-secondary-inverted</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="multiple-secondaries">
+ <output-dir compare="Text">multiple-secondaries</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="multiple-correlated-secondaries">
+ <output-dir compare="Text">multiple-secondaries</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="upsert-case-returning">
+ <output-dir compare="Text">upsert-case-returning</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="json-parser">
+ <test-case FilePath="json-parser">
+ <compilation-unit name="numeric-tinyint">
+ <output-dir compare="Text">numeric-tinyint</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="numeric-tinyint-overflow">
+ <output-dir compare="Text">numeric-tinyint</output-dir>
+ <expected-error>Numeric value (1000) out of range of Java byte</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="numeric-float">
+ <output-dir compare="Text">numeric-float</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="numeric-demote-double-bigint">
+ <output-dir compare="Text">numeric-demote-double-bigint</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="numeric-promote-bigint-double">
+ <output-dir compare="Text">numeric-promote-bigint-double</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="duplicate-fields">
+ <output-dir compare="Text">duplicate-fields</output-dir>
+ <expected-error>Duplicate field 'field'</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="malformed-json">
+ <output-dir compare="Text">malformed-json</output-dir>
+ <expected-error>Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="null-missing">
+ <output-dir compare="Text">null-missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="nonoptional-missing">
+ <output-dir compare="Text">nonoptional-missing</output-dir>
+ <expected-error>ASX3075: Closed field missing_value has null value</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="nonoptional-null">
+ <output-dir compare="Text">nonoptional-null</output-dir>
+ <expected-error>ASX3075: Closed field null_value has null value</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="spatial">
+ <output-dir compare="Text">spatial</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="spatial-line-3-points">
+ <output-dir compare="Text">spatial-line-3-points</output-dir>
+ <expected-error>Line must have 4 coordinates</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="spatial-polygon-unclosed">
+ <output-dir compare="Text">spatial-polygon-unclosed</output-dir>
+ <expected-error>Unclosed polygon is not supported</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="spatial-polygon-with-hole">
+ <output-dir compare="Text">spatial-polygon-with-hole</output-dir>
+ <expected-error>Only simple geometries are supported (Point, LineString and Polygon without holes)</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="temporal">
+ <output-dir compare="Text">temporal</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="type-mismatch">
+ <output-dir compare="Text">type-mismatch</output-dir>
+ <expected-error>ASX3054: Mismatch Type, expecting a value of type string</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="unsupported-type-circle">
+ <output-dir compare="Text">unsupported-type-circle</output-dir>
+ <expected-error>ASX0004: Unsupported type: org.apache.asterix.external.parser.factory.JSONDataParserFactory cannot process input type circle</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="unsupported-type-daytimeduration">
+ <output-dir compare="Text">unsupported-type-daytimeduration</output-dir>
+ <expected-error>ASX0004: Unsupported type: org.apache.asterix.external.parser.factory.JSONDataParserFactory cannot process input type daytimeduration</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="unsupported-type-duration">
+ <output-dir compare="Text">unsupported-type-duration</output-dir>
+ <expected-error>ASX0004: Unsupported type: org.apache.asterix.external.parser.factory.JSONDataParserFactory cannot process input type duration</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="unsupported-type-interval">
+ <output-dir compare="Text">unsupported-type-interval</output-dir>
+ <expected-error>ASX0004: Unsupported type: org.apache.asterix.external.parser.factory.JSONDataParserFactory cannot process input type interval</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="unsupported-type-multiset">
+ <output-dir compare="Text">unsupported-type-multiset</output-dir>
+ <expected-error>ASX0004: Unsupported type: org.apache.asterix.external.parser.factory.JSONDataParserFactory cannot process input type multiset</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="unsupported-type-point3d">
+ <output-dir compare="Text">unsupported-type-point3d</output-dir>
+ <expected-error>ASX0004: Unsupported type: org.apache.asterix.external.parser.factory.JSONDataParserFactory cannot process input type point3d</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser">
+ <compilation-unit name="unsupported-type-rectangle">
+ <output-dir compare="Text">unsupported-type-rectangle</output-dir>
+ <expected-error>ASX0004: Unsupported type: org.apache.asterix.external.parser.factory.JSONDataParserFactory cannot process input type rectangle</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="json-parser" check-warnings="true">
+ <compilation-unit name="parse-json-function">
+ <output-dir compare="Text">parse-json-function</output-dir>
+ <source-location>false</source-location>
+ <expected-warn>Type mismatch: function parse-json expects its 1st input parameter to be of type string, but the actual input type is bigint</expected-warn>
+ <expected-warn>Malformed input stream</expected-warn>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="composite-key">
+ <test-case FilePath="composite-key">
+ <compilation-unit name="query-ASTERIXDB-920">
+ <output-dir compare="Text">query-ASTERIXDB-920</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="composite-key">
+ <compilation-unit name="query-ASTERIXDB-2334">
+ <output-dir compare="Text">query-ASTERIXDB-2334</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="composite-key">
+ <compilation-unit name="composite-low-high">
+ <output-dir compare="Text">composite-low-high</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="composite-key">
+ <compilation-unit name="composite-prefix">
+ <output-dir compare="Text">composite-prefix</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="composite-key">
+ <compilation-unit name="composite-prefix-low-high">
+ <output-dir compare="Text">composite-prefix-low-high</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="limit">
+ <test-case FilePath="limit">
+ <compilation-unit name="limit_negative_value">
+ <output-dir compare="Text">limit_negative_value</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="limit_type_01">
+ <output-dir compare="Text">limit_type_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="limit_type_02">
+ <output-dir compare="Text">limit_type_01</output-dir>
+ <expected-error>ASX0039: Expected integer value, got 2.75 (in line 28, at column 10)</expected-error>
+ <expected-error>ASX0039: Expected integer value, got 1.75 (in line 28, at column 19)</expected-error>
+ <expected-error>ASX1091: Type mismatch: expected value of type integer, but got the value of type string (in line 28, at column 7)</expected-error>
+ <expected-error>ASX1091: Type mismatch: expected value of type integer, but got the value of type boolean (in line 28, at column 16)</expected-error>
+ <expected-error>ASX0021: Source value 9999999999 is out of range that integer can hold - integer.MAX_VALUE: 2147483647, integer.MIN_VALUE: -2147483648</expected-error>
+ <expected-error>ASX0021: Source value 8888888888 is out of range that integer can hold - integer.MAX_VALUE: 2147483647, integer.MIN_VALUE: -2147483648</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="offset_without_limit">
+ <output-dir compare="Text">offset_without_limit</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-external-scan">
+ <output-dir compare="Text">push-limit-to-external-scan</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-external-scan-select">
+ <output-dir compare="Text">push-limit-to-external-scan-select</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-primary-scan">
+ <output-dir compare="Text">push-limit-to-primary-scan</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-primary-scan-select">
+ <output-dir compare="Text">push-limit-to-primary-scan-select</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-primary-lookup">
+ <output-dir compare="Text">push-limit-to-primary-lookup</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="push-limit-to-primary-lookup-select">
+ <output-dir compare="Text">push-limit-to-primary-lookup-select</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="query-ASTERIXDB-2420">
+ <output-dir compare="Text">query-ASTERIXDB-2420</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="compression">
+ <test-case FilePath="compression">
+ <compilation-unit name="incompressible-pages/large-page">
+ <output-dir compare="Text">incompressible-pages/large-page</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="compression">
+ <compilation-unit name="incompressible-pages/small-page">
+ <output-dir compare="Text">incompressible-pages/small-page</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="compression">
+ <compilation-unit name="invalid-compression-scheme">
+ <output-dir compare="Text">invalid-compression-scheme</output-dir>
+ <expected-error>ASX1096: Unknown compression scheme zip. Supported schemes are [snappy,none]</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="compression">
+ <compilation-unit name="scheme-none">
+ <output-dir compare="Text">scheme-none</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="compression">
+ <compilation-unit name="scheme-snappy">
+ <output-dir compare="Text">scheme-snappy</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="ddl-with-clause">
+ <test-case FilePath="ddl-with-clause">
+ <compilation-unit name="missing-non-optional">
+ <output-dir compare="Text">missing-non-optional</output-dir>
+ <expected-error>ASX1061: Field 'merge-policy.name' in the with clause cannot be null or missing</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl-with-clause">
+ <compilation-unit name="type-mismatch">
+ <output-dir compare="Text">type-mismatch</output-dir>
+ <expected-error>ASX1060: Field 'merge-policy.parameters.max-mergable-component-size' in the with clause must be of type bigint, but found string</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl-with-clause">
+ <compilation-unit name="unsupported-field">
+ <output-dir compare="Text">unsupported-field</output-dir>
+ <expected-error>ASX1059: Field(s) [unknown-field] unsupported in the with clause</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl-with-clause">
+ <compilation-unit name="unsupported-subfield">
+ <output-dir compare="Text">unsupported-subfield</output-dir>
+ <expected-error>ASX1097: Subfield(s) [unknown-subfield] in 'merge-policy' unsupported in the with clause</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="metrics">
+ <test-case FilePath="metrics">
+ <compilation-unit name="full-scan">
+ <output-dir compare="Text">full-scan</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="warnings">
+ <test-case FilePath="warnings" check-warnings="true">
+ <compilation-unit name="inapplicable-hint-warning">
+ <output-dir compare="Text">inapplicable-hint-warning</output-dir>
+ <expected-warn>HYR10006: Could not apply Group By hint: hash</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: indexnl. "hash" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: hash. "hash-bcast", "indexnl", "hashjoin", "skip-index", "use-index", "selectivity", "productivity" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: auto. "indexnl", "skip-index", "hashjoin", "use-index", "selectivity" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: hash. "indexnl", "range", "hashjoin", "skip-index", "spatial-partitioning", "use-index" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: hash. None expected at this location</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="warnings" check-warnings="true">
+ <compilation-unit name="min-max-incompatible-types">
+ <output-dir compare="Text">min-max-incompatible-types</output-dir>
+ <expected-warn>ASX0003: Type incompatibility: function min/max gets incompatible input values: bigint and string</expected-warn>
+ <expected-warn>ASX0004: Unsupported type: min/max cannot process input type object</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="warnings" check-warnings="true">
+ <compilation-unit name="plan-warning">
+ <output-dir compare="Text">plan-warning</output-dir>
+ <expected-warn>HYR10007: Encountered a cross product join</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="warnings" check-warnings="true">
+ <compilation-unit name="unknown-hint-warning">
+ <output-dir compare="Text">unknown-hint-warning</output-dir>
+ <expected-warn>ASX1107: Unexpected hint: unknown_hint_groupby. "hash" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: unknown_hint_relexpr. "hash-bcast", "indexnl", "hashjoin", "skip-index", "use-index", "selectivity", "productivity" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: unknown_hint_between. "indexnl", "skip-index", "hashjoin", "use-index", "selectivity" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: unknown_hint_funcall. "indexnl", "range", "hashjoin", "skip-index", "spatial-partitioning", "use-index" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: unknown_hint_elsewhere. None expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: unknown_hint_relexpr_6. "hash-bcast", "indexnl", "hashjoin", "skip-index", "use-index", "selectivity", "productivity" expected at this location</expected-warn>
+ <expected-warn>ASX1107: Unexpected hint: unknown_hint_relexpr_6. "hash-bcast", "indexnl", "hashjoin", "skip-index", "use-index", "selectivity", "productivity" expected at this location</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <!--test-case FilePath="warnings">
+ <compilation-unit name="warnings-limit">
+ <output-dir compare="Clean-JSON">warnings-limit</output-dir>
+ </compilation-unit>
+ </test-case!-->
+ </test-group>
+ <test-group name="nonpure">
+ <test-case FilePath="nonpure">
+ <compilation-unit name="global-datetime-use-index">
+ <output-dir compare="Text">global-datetime-use-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="nonpure">
+ <compilation-unit name="local-datetime-ignore-index">
+ <output-dir compare="Text">local-datetime-ignore-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="orderby_limit">
+ <test-case FilePath="orderby_limit">
+ <compilation-unit name="limit_on_variable_01">
+ <output-dir compare="Text">limit_on_variable_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="orderby_limit">
+ <compilation-unit name="orderby_limit_01">
+ <output-dir compare="Text">orderby_limit_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="orderby_limit">
+ <compilation-unit name="orderby_limit_02">
+ <output-dir compare="Text">orderby_limit_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="orderby_limit">
+ <compilation-unit name="orderby_limit_offset_01">
+ <output-dir compare="Text">orderby_limit_offset_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="orderby_limit">
+ <compilation-unit name="orderby_limit_primary_index_01">
+ <output-dir compare="Text">orderby_limit_primary_index_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="fun_return_null_missing/string_fun">
+ <test-case FilePath="fun_return_null_missing/string_fun" check-warnings="true">
+ <compilation-unit name="string_fun_001">
+ <output-dir compare="Text">string_fun_001</output-dir>
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 41, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function uppercase expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 38, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 42, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 33, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-length expects its 1st input parameter to be of type string, but the actual input type is integer (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is integer (in line 40, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function lowercase expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 45, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function contains expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 34, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 43, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function position expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 46, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-to-codepoint expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 35, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function initcap expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 39, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is integer (in line 44, at column 1)</expected-warn>
+
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 41, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function uppercase expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 38, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 42, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 33, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-length expects its 1st input parameter to be of type string, but the actual input type is integer (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is integer (in line 40, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function lowercase expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 45, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function contains expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 34, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 43, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function position expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 46, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-to-codepoint expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 35, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function initcap expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 39, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is integer (in line 44, at column 1)</expected-warn>
+
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 41, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function uppercase expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 38, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 42, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 33, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-length expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 40, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function lowercase expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 45, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function contains expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 34, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 43, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function position expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 46, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-to-codepoint expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 35, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function initcap expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 39, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 44, at column 1)</expected-warn>
+
+ <expected-warn>Type mismatch: function regexp-position expects its 1st input parameter to be of type string, but the actual input type is integer (in line 38, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function replace expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 42, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function starts-with expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function reverse expects its 1st input parameter to be of type string, but the actual input type is integer (in line 43, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-equal expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 41, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-after expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 44, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 2nd input parameter to be of type string, but the actual input type is integer (in line 34, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-position expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 39, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-before expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 45, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-replace expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 40, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 35, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function split expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 46, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ends-with expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 33, at column 1)</expected-warn>
+
+ <expected-warn>Type mismatch: function regexp-position expects its 1st input parameter to be of type string, but the actual input type is integer (in line 38, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function replace expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 42, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function starts-with expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function reverse expects its 1st input parameter to be of type string, but the actual input type is integer (in line 43, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-equal expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 41, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-after expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 44, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 2nd input parameter to be of type string, but the actual input type is integer (in line 34, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-position expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 39, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-before expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 45, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-replace expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 40, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 35, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function split expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 46, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ends-with expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 33, at column 1)</expected-warn>
+
+ <expected-warn>Type mismatch: function regexp-position expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 38, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function replace expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 42, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function starts-with expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function reverse expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 43, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 3rd input parameter to be of type string, but the actual input type is bigint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-equal expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 41, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-after expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 44, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 34, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-position expects its 3rd input parameter to be of type string, but the actual input type is bigint (in line 39, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-before expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 45, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-replace expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 40, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 3rd input parameter to be of type string, but the actual input type is bigint (in line 35, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function split expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 46, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ends-with expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 33, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fun_return_null_missing/string_fun" check-warnings="true">
+ <compilation-unit name="string_fun_002">
+ <output-dir compare="Text">string_fun_002</output-dir>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 42, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-replace expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 52, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 48, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function position expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 43, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 2nd input parameter to be of type string, but the actual input type is integer (in line 46, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function replace expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 54, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function starts-with expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 44, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-position expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 51, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function rtrim expects its 1st input parameter to be of type string, but the actual input type is integer (in line 41, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-after expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 56, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring-before expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 57, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function lowercase expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 34, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ends-with expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 45, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 38, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function uppercase expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 35, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-like expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 49, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function matches expects its 3rd input parameter to be of type string, but the actual input type is tinyint (in line 47, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-equal expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 53, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function split expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 58, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-to-codepoint expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function trim expects its 1st input parameter to be of type string, but the actual input type is integer (in line 37, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function reverse expects its 1st input parameter to be of type string, but the actual input type is integer (in line 55, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 29, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function contains expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 31, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 1st input parameter to be of type string, but the actual input type is smallint (in line 39, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function initcap expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function ltrim expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 40, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-length expects its 1st input parameter to be of type string, but the actual input type is integer (in line 33, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function like expects its 1st input parameter to be of type string, but the actual input type is tinyint (in line 30, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function regexp-position expects its 1st input parameter to be of type string, but the actual input type is integer (in line 50, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fun_return_null_missing/string_fun" check-warnings="true">
+ <compilation-unit name="string_fun_003">
+ <output-dir compare="Text">string_fun_003</output-dir>
+ <expected-warn>Invalid value: function repeat expects its 2nd input parameter to be an integer value, got 5.3 (in line 31, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function regexp-replace expects its 4th input parameter to be an integer value, got 5.3 (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function replace expects its 4th input parameter to be an integer value, got 5.3 (in line 34, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function repeat expects its 2nd input parameter to be a non-negative value, got -2.0 (in line 30, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-join expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 32, at column 1)</expected-warn>
+
+ <expected-warn>Invalid value: function repeat expects its 2nd input parameter to be an integer value, got 5.3 (in line 31, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function regexp-replace expects its 4th input parameter to be an integer value, got 5.3 (in line 36, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 37, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function replace expects its 4th input parameter to be an integer value, got 5.3 (in line 34, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function repeat expects its 2nd input parameter to be a non-negative value, got -2.0 (in line 30, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-join expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 32, at column 1)</expected-warn>
+
+ <expected-warn>Invalid value: function repeat expects its 2nd input parameter to be an integer value, got Infinity (in line 29, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function replace expects its 4th input parameter to be an integer value, got -Infinity (in line 30, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function replace expects its 4th input parameter to be an integer value, got NaN (in line 31, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function regexp-replace expects its 4th input parameter to be an integer value, got NaN (in line 32, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function regexp-replace expects its 4th input parameter to be an integer value, got -Infinity (in line 33, at column 1)</expected-warn>
+ <expected-warn>Invalid value: function substring expects its 2nd input parameter to be an integer value, got Infinity (in line 34, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fun_return_null_missing/string_fun" check-warnings="true">
+ <compilation-unit name="string_fun_004">
+ <output-dir compare="Text">string_fun_004</output-dir>
+ <expected-warn>Type mismatch: function string-concat expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 30, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 33, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-concat expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 31, at column 7)</expected-warn>
+ <expected-warn>Type mismatch: function codepoint-to-string expects its 1st input parameter to be of type array, but the actual input type is string (in line 34, at column 1)</expected-warn>
+ <expected-warn>Unsupported type: codepoint-to-string cannot process input type string (in line 35, at column 1)</expected-warn>
+
+ <expected-warn>Type mismatch: function string-concat expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 30, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function substring expects its 1st input parameter to be of type string, but the actual input type is bigint (in line 33, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-concat expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 31, at column 7)</expected-warn>
+ <expected-warn>Type mismatch: function codepoint-to-string expects its 1st input parameter to be of type array, but the actual input type is string (in line 34, at column 1)</expected-warn>
+ <expected-warn>Unsupported type: codepoint-to-string cannot process input type string (in line 35, at column 1)</expected-warn>
+
+ <expected-warn>Type mismatch: function string-concat expects its 2nd input parameter to be of type string, but the actual input type is bigint (in line 30, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function string-concat expects its 2nd input parameter to be of type string, but the actual input type is tinyint (in line 31, at column 7)</expected-warn>
+ <expected-warn>Type mismatch: function substring expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string (in line 32, at column 1)</expected-warn>
+ <expected-warn>Type mismatch: function codepoint-to-string expects its 1st input parameter to be of type array, but the actual input type is string (in line 33, at column 1)</expected-warn>
+ <expected-warn>Unsupported type: codepoint-to-string cannot process input type string (in line 34, at column 1)</expected-warn>
+ <expected-warn>Unsupported type: codepoint-to-string cannot process input type double (in line 35, at column 1)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="fun_return_null_missing/numeric_fun" >
+ <test-case FilePath="fun_return_null_missing/numeric_fun" check-warnings="true">
+ <compilation-unit name="numeric_fun_001">
+ <output-dir compare="Text">numeric_fun_001</output-dir>
+ <expected-warn>Type mismatch: function abs expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function acos expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string </expected-warn>
+ <expected-warn>Type mismatch: function asin expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function atan expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function atan2 expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function degrees expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function radians expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function cos expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function cosh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sin expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sinh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function tan expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function tanh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function exp expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function ln expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function log expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sqrt expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sign expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function ceiling expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function floor expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-unary-minus expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function random expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+
+ <expected-warn>Type mismatch: function abs expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function acos expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string </expected-warn>
+ <expected-warn>Type mismatch: function asin expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function atan expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function atan2 expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function degrees expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function radians expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function cos expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function cosh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sin expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sinh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function tan expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function tanh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function exp expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function ln expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function log expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sqrt expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sign expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function ceiling expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function floor expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-unary-minus expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function random expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+
+ <expected-warn>Type mismatch: function abs expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function acos expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string </expected-warn>
+ <expected-warn>Type mismatch: function asin expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function atan expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function atan2 expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function degrees expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function radians expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function cos expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function cosh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sin expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sinh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function tan expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function tanh expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function exp expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function ln expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function log expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sqrt expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function sign expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function ceiling expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function floor expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-unary-minus expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function random expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fun_return_null_missing/numeric_fun" check-warnings="true">
+ <compilation-unit name="numeric_fun_002">
+ <output-dir compare="Text">numeric_fun_002</output-dir>
+ <expected-warn>Type mismatch: function round expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function trunc expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function trunc expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round expects its 2nd input parameter to be of type tinyint, smallint, integer or bigint, but the actual input type is string</expected-warn>
+
+ <expected-warn>Type mismatch: function round expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function trunc expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function trunc expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string</expected-warn>
+
+ <expected-warn>Type mismatch: function round expects its 2nd input parameter to be of type tinyint, smallint, integer or bigint, but the actual input type is string (in line 33, at column 24)</expected-warn>
+ <expected-warn>Type mismatch: function round expects its 2nd input parameter to be of type tinyint, smallint, integer or bigint, but the actual input type is string (in line 33, at column 47)</expected-warn>
+ <expected-warn>Type mismatch: function trunc expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string (in line 34, at column 24)</expected-warn>
+ <expected-warn>Type mismatch: function trunc expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string (in line 34, at column 47)</expected-warn>
+
+ <expected-warn>Type mismatch: function round-half-to-even expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string (in line 36, at column 40)</expected-warn>
+ <expected-warn>Type mismatch: function round-half-to-even expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string (in line 36, at column 79)</expected-warn>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fun_return_null_missing/numeric_fun" check-warnings="true">
+ <compilation-unit name="numeric_fun_003">
+ <output-dir compare="Text">numeric_fun_003</output-dir>
+ <expected-warn>Type mismatch: function numeric-add expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-divide expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-multiply gets incompatible input values: daytimeduration and string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-add expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-divide gets incompatible input values: time and string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-add gets incompatible input values: yearmonthduration and string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-multiply expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-add gets incompatible input values: daytimeduration and string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-div expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function power expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-mod expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-add gets incompatible input values: time and string</expected-warn>
+ <expected-warn>Type incompatibility: function power gets incompatible input values: yearmonthduration and string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-multiply expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-multiply expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-mod expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-divide expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-div expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-multiply gets incompatible input values: date and string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-add expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-add gets incompatible input values: duration and string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-div gets incompatible input values: datetime and string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-mod expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function power expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-add gets incompatible input values: date and string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-mod gets incompatible input values: duration and string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-divide expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-div expects its 2nd input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function numeric-subtract expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <expected-warn>Type incompatibility: function numeric-add gets incompatible input values: datetime and string</expected-warn>
+ <expected-warn>Type mismatch: function power expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float, double, date, time, datetime, duration, yearmonthduration or daytimeduration, but the actual input type is string</expected-warn>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fun_return_null_missing" check-warnings="true">
+ <compilation-unit name="field-access">
+ <output-dir compare="Text">field-access</output-dir>
+ <expected-warn>Type mismatch: function field-access-by-name expects its 1st input parameter to be of type object, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function field-access-by-name expects its 1st input parameter to be of type object, but the actual input type is string</expected-warn>
+ <expected-warn>Type mismatch: function field-access-by-name expects its 1st input parameter to be of type object, but the actual input type is string</expected-warn>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="window">
+ <test-case FilePath="window">
+ <compilation-unit name="cume_dist_01">
+ <output-dir compare="Text">cume_dist_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="dense_rank_01">
+ <output-dir compare="Text">dense_rank_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="first_value_01">
+ <output-dir compare="Text">first_value_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="lag_01">
+ <output-dir compare="Text">lag_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="last_value_01">
+ <output-dir compare="Text">last_value_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="lead_01">
+ <output-dir compare="Text">lead_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="misc_01">
+ <output-dir compare="Text">misc_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="nth_value_01">
+ <output-dir compare="Text">nth_value_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="ntile_01">
+ <output-dir compare="Text">ntile_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="percent_rank_01">
+ <output-dir compare="Text">percent_rank_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="pg_win">
+ <output-dir compare="Text">pg_win</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="rank_01">
+ <output-dir compare="Text">rank_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="ratio_to_report_01">
+ <output-dir compare="Text">ratio_to_report_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="row_number_01">
+ <output-dir compare="Text">row_number_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="win_negative">
+ <output-dir compare="Text">misc_01</output-dir>
+ <expected-error>ASX0002: Type mismatch</expected-error>
+ <expected-error>ASX1104: Invalid modifier FROM FIRST/LAST for function</expected-error>
+ <expected-error>ASX1037: Invalid query parameter compiler.windowmemory</expected-error>
+ <expected-error>ASX1102: Expected window or aggregate function, got: lowercase</expected-error>
+ <expected-error>ASX1079: Compilation error: count is a SQL-92 aggregate function</expected-error>
+ <expected-error>ASX1104: Invalid modifier RESPECT/IGNORE NULLS for function</expected-error>
+ <expected-error>ASX1104: Invalid modifier RESPECT/IGNORE NULLS for function</expected-error>
+ <expected-error>ASX1104: Invalid modifier FROM FIRST/LAST for function</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="win_null_missing">
+ <output-dir compare="Text">win_null_missing</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="win_opt_01">
+ <output-dir compare="Text">win_opt_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="window">
+ <compilation-unit name="win_opt_02">
+ <output-dir compare="Text">win_opt_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-benchmark/pom.xml b/asterixdb/asterix-benchmark/pom.xml
index cffee2e..f56718e 100644
--- a/asterixdb/asterix-benchmark/pom.xml
+++ b/asterixdb/asterix-benchmark/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-benchmark</artifactId>
diff --git a/asterixdb/asterix-client-helper/pom.xml b/asterixdb/asterix-client-helper/pom.xml
index 32f0722..bd9a81a 100644
--- a/asterixdb/asterix-client-helper/pom.xml
+++ b/asterixdb/asterix-client-helper/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<properties>
<root.dir>${basedir}/..</root.dir>
diff --git a/asterixdb/asterix-column/pom.xml b/asterixdb/asterix-column/pom.xml
new file mode 100644
index 0000000..ef87c91
--- /dev/null
+++ b/asterixdb/asterix-column/pom.xml
@@ -0,0 +1,151 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-asterixdb</artifactId>
+ <groupId>org.apache.asterix</groupId>
+ <version>0.9.9-SNAPSHOT</version>
+ </parent>
+ <artifactId>asterix-column</artifactId>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+
+ <properties>
+ <root.dir>${basedir}/..</root.dir>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <configuration>
+ <licenses>
+ <license implementation="org.apache.rat.analysis.license.ApacheSoftwareLicense20"/>
+ </licenses>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/result/**</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-om</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-runtime</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-external-data</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree-column</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>algebricks-data</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-column</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-encoding</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/ColumnManager.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/ColumnManager.java
new file mode 100644
index 0000000..c1402c8
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/ColumnManager.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.operation.lsm.merge.MergeColumnTupleProjector;
+import org.apache.asterix.column.operation.lsm.merge.MergeColumnWriteMetadata;
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.column.values.IColumnValuesWriterFactory;
+import org.apache.asterix.column.values.reader.ColumnValueReaderFactory;
+import org.apache.asterix.column.values.writer.ColumnValuesWriterFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManager;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+
+public final class ColumnManager implements IColumnManager {
+ private final ARecordType datasetType;
+ private final ARecordType metaType;
+ private final List<List<String>> primaryKeys;
+ private final List<Integer> keySourceIndicator;
+ private final MergeColumnTupleProjector mergeColumnTupleProjector;
+
+ ColumnManager(ARecordType datasetType, ARecordType metaType, List<List<String>> primaryKeys,
+ List<Integer> keySourceIndicator) {
+ this.datasetType = datasetType;
+ this.metaType = metaType;
+ this.primaryKeys = primaryKeys;
+ this.keySourceIndicator = keySourceIndicator;
+ IColumnValuesReaderFactory readerFactory = new ColumnValueReaderFactory();
+ mergeColumnTupleProjector =
+ new MergeColumnTupleProjector(datasetType, metaType, primaryKeys.size(), readerFactory);
+ }
+
+ @Override
+ public IColumnMetadata activate() throws HyracksDataException {
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef = new MutableObject<>();
+ IColumnValuesWriterFactory factory = new ColumnValuesWriterFactory(multiPageOpRef);
+ return new FlushColumnMetadata(datasetType, metaType, primaryKeys, keySourceIndicator, factory, multiPageOpRef);
+ }
+
+ @Override
+ public IColumnMetadata activate(IValueReference metadata) throws HyracksDataException {
+ try {
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef = new MutableObject<>();
+ IColumnValuesWriterFactory writerFactory = new ColumnValuesWriterFactory(multiPageOpRef);
+ return FlushColumnMetadata.create(datasetType, metaType, primaryKeys, keySourceIndicator, writerFactory,
+ multiPageOpRef, metadata);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public IColumnMetadata createMergeColumnMetadata(IValueReference metadata,
+ List<IColumnTupleIterator> componentsTuples) throws HyracksDataException {
+ try {
+ return MergeColumnWriteMetadata.create(datasetType, metaType, primaryKeys.size(), new MutableObject<>(),
+ metadata, componentsTuples);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+
+ }
+
+ @Override
+ public IColumnTupleProjector getMergeColumnProjector() {
+ return mergeColumnTupleProjector;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/ColumnManagerFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/ColumnManagerFactory.java
new file mode 100644
index 0000000..8e35416
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/ColumnManagerFactory.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnTupleReaderWriterFactory;
+import org.apache.asterix.column.operation.lsm.load.LoadColumnTupleReaderWriterFactory;
+import org.apache.asterix.column.operation.lsm.merge.MergeColumnTupleReaderWriterFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReaderWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManager;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManagerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public final class ColumnManagerFactory implements IColumnManagerFactory {
+ private static final long serialVersionUID = -5003159552950739235L;
+ private final ARecordType datasetType;
+ private final ARecordType metaType;
+ private final List<List<String>> primaryKeys;
+ private final List<Integer> keySourceIndicator;
+ private final int pageSize;
+ private final int maxTupleCount;
+ private final float tolerance;
+
+ public ColumnManagerFactory(ARecordType datasetType, ARecordType metaType, List<List<String>> primaryKeys,
+ List<Integer> keySourceIndicator, int pageSize, int maxTupleCount, float tolerance) {
+ this.pageSize = pageSize;
+ this.maxTupleCount = maxTupleCount;
+ this.tolerance = tolerance;
+
+ this.datasetType = datasetType;
+ if (keySourceIndicator.size() != 1) {
+ throw new UnsupportedOperationException(
+ "Primary keys split between meta-type and datasetType is not supported");
+ }
+ this.keySourceIndicator = keySourceIndicator;
+ this.metaType = metaType;
+ this.primaryKeys = primaryKeys;
+ }
+
+ @Override
+ public IColumnManager createColumnManager() {
+ return new ColumnManager(datasetType, metaType, primaryKeys, keySourceIndicator);
+ }
+
+ @Override
+ public AbstractColumnTupleReaderWriterFactory getLoadColumnTupleReaderWriterFactory() {
+ return new LoadColumnTupleReaderWriterFactory(pageSize, maxTupleCount, tolerance);
+ }
+
+ @Override
+ public AbstractColumnTupleReaderWriterFactory getFlushColumnTupleReaderWriterFactory() {
+ return new FlushColumnTupleReaderWriterFactory(pageSize, maxTupleCount, tolerance);
+ }
+
+ @Override
+ public AbstractColumnTupleReaderWriterFactory createMergeColumnTupleReaderWriterFactory() {
+ return new MergeColumnTupleReaderWriterFactory(pageSize, maxTupleCount, tolerance);
+ }
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ ObjectNode json = registry.getClassIdentifier(getClass(), serialVersionUID);
+ json.putPOJO("datasetType", datasetType.toJson(registry));
+ if (metaType != null) {
+ json.putPOJO("metaType", metaType.toJson(registry));
+ }
+
+ json.put("pageSize", pageSize);
+ json.put("maxTupleCount", maxTupleCount);
+ json.put("tolerance", tolerance);
+
+ ArrayNode primaryKeysArray = json.putArray("primaryKeys");
+ for (List<String> primaryKey : primaryKeys) {
+ ArrayNode primaryKeyArray = primaryKeysArray.addArray();
+ for (String path : primaryKey) {
+ primaryKeyArray.add(path);
+ }
+ }
+
+ ArrayNode keySourceIndicatorNode = json.putArray("keySourceIndicator");
+ for (int keySourceIndicatorInt : keySourceIndicator) {
+ keySourceIndicatorNode.add(keySourceIndicatorInt);
+ }
+ return json;
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ ARecordType datasetType = (ARecordType) registry.deserialize(json.get("datasetType"));
+ JsonNode metaItemTypeNode = json.get("metaType");
+ ARecordType metaType = null;
+ if (metaItemTypeNode != null) {
+ metaType = (ARecordType) registry.deserialize(metaItemTypeNode);
+ }
+
+ int pageSize = json.get("pageSize").asInt();
+ int maxTupleCount = json.get("maxTupleCount").asInt();
+ float tolerance = (float) json.get("tolerance").asDouble();
+
+ List<List<String>> primaryKeys = new ArrayList<>();
+ ArrayNode primaryKeysNode = (ArrayNode) json.get("primaryKeys");
+ for (int i = 0; i < primaryKeysNode.size(); i++) {
+ List<String> primaryKey = new ArrayList<>();
+ ArrayNode primaryKeyNode = (ArrayNode) primaryKeysNode.get(i);
+ for (int j = 0; j < primaryKeyNode.size(); j++) {
+ primaryKey.add(primaryKeyNode.get(i).asText());
+ }
+ primaryKeys.add(primaryKey);
+ }
+
+ List<Integer> keySourceIndicator = new ArrayList<>();
+ ArrayNode keySourceIndicatorNode = (ArrayNode) json.get("keySourceIndicator");
+ for (int i = 0; i < keySourceIndicatorNode.size(); i++) {
+ keySourceIndicator.add(keySourceIndicatorNode.get(i).asInt());
+ }
+
+ return new ColumnManagerFactory(datasetType, metaType, primaryKeys, keySourceIndicator, pageSize, maxTupleCount,
+ tolerance);
+ }
+
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractNestedValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractNestedValueAssembler.java
new file mode 100644
index 0000000..1a4c3ef
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractNestedValueAssembler.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+abstract class AbstractNestedValueAssembler extends AbstractValueAssembler {
+ protected final ArrayBackedValueStorage storage;
+
+ AbstractNestedValueAssembler(int level, AssemblerInfo info) {
+ super(level, info);
+ storage = new ArrayBackedValueStorage();
+ }
+
+ /**
+ * @return whether the nested assembler was started or not
+ */
+ final boolean isStarted() {
+ return started;
+ }
+
+ /**
+ * Add a nested value
+ *
+ * @param value contains the value and its information
+ */
+ abstract void addValue(AbstractValueAssembler value) throws HyracksDataException;
+
+ /**
+ * Add a nested {@link ATypeTag#NULL}
+ *
+ * @param value contains the value's information
+ */
+ abstract void addNull(AbstractValueAssembler value) throws HyracksDataException;
+
+ /**
+ * Add a nested {@link ATypeTag#MISSING}
+ */
+ void addMissing() throws HyracksDataException {
+ //By default, we ignore missing
+ }
+
+ @Override
+ final void addNullToAncestor(int nullLevel) throws HyracksDataException {
+ AbstractNestedValueAssembler parent = getParent();
+ if (nullLevel + 1 == level) {
+ parent.start();
+ parent.addNull(this);
+ return;
+ }
+ parent.addNullToAncestor(nullLevel);
+ }
+
+ @Override
+ final void addMissingToAncestor(int missingLevel) throws HyracksDataException {
+ AbstractNestedValueAssembler parent = getParent();
+ if (missingLevel + 1 == level) {
+ parent.start();
+ parent.addMissing();
+ return;
+ }
+ parent.addMissingToAncestor(missingLevel);
+ }
+
+ /**
+ * Recursively start the path of this assembler by staring all un-started parents
+ */
+ public final void start() {
+ if (started) {
+ return;
+ }
+ started = true;
+ reset();
+ AbstractNestedValueAssembler parent = getParent();
+ if (parent != null && !parent.isStarted()) {
+ parent.start();
+ }
+ }
+
+ /**
+ * End the assembler and add this nested value to its parent
+ */
+ public final void end() throws HyracksDataException {
+ if (started) {
+ addValueToParent();
+ started = false;
+ }
+
+ if (isDelegate()) {
+ getParent().end();
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractPrimitiveValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractPrimitiveValueAssembler.java
new file mode 100644
index 0000000..9f1809d
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractPrimitiveValueAssembler.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.column.assembler.value.IValueGetter;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public abstract class AbstractPrimitiveValueAssembler extends AbstractValueAssembler {
+ /**
+ * An indicator to go to the next value
+ */
+ public static final int NEXT_ASSEMBLER = -1;
+ protected final IValueGetter primitiveValueGetter;
+ protected final IColumnValuesReader reader;
+
+ AbstractPrimitiveValueAssembler(int level, AssemblerInfo info, IColumnValuesReader reader,
+ IValueGetter primitiveValueGetter) {
+ super(level, info);
+ this.primitiveValueGetter = primitiveValueGetter;
+ this.reader = reader;
+ }
+
+ public final void reset(AbstractBytesInputStream in, int startIndex, int numberOfTuples)
+ throws HyracksDataException {
+ reader.reset(in, numberOfTuples);
+ reader.skip(startIndex);
+ }
+
+ @Override
+ public final IValueReference getValue() throws HyracksDataException {
+ return primitiveValueGetter.getValue(reader);
+ }
+
+ @Override
+ void addNullToAncestor(int nullLevel) throws HyracksDataException {
+ AbstractNestedValueAssembler parent = getParent();
+ if (nullLevel + 1 == level) {
+ parent.start();
+ parent.addNull(this);
+ return;
+ }
+ parent.addNullToAncestor(nullLevel);
+ }
+
+ @Override
+ void addMissingToAncestor(int missingLevel) throws HyracksDataException {
+ AbstractNestedValueAssembler parent = getParent();
+ if (missingLevel + 1 == level) {
+ parent.start();
+ parent.addMissing();
+ return;
+ }
+ parent.addMissingToAncestor(missingLevel);
+ }
+
+ @Override
+ final void addValueToParent() throws HyracksDataException {
+ AbstractNestedValueAssembler parent = getParent();
+ parent.start();
+ getParent().addValue(this);
+ }
+
+ public final int getColumnIndex() {
+ return reader.getColumnIndex();
+ }
+
+ public final void skip(int count) throws HyracksDataException {
+ reader.skip(count);
+ }
+
+ /**
+ * Move to the next primitive value assembler
+ *
+ * @return the index of the next value
+ */
+ public abstract int next() throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractValueAssembler.java
new file mode 100644
index 0000000..0071917
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractValueAssembler.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+public abstract class AbstractValueAssembler {
+ protected static final VoidPointable NULL;
+ protected static final VoidPointable MISSING;
+ private final AbstractNestedValueAssembler parent;
+ private final IValueReference fieldName;
+ private final int fieldIndex;
+ private final boolean delegate;
+ protected final int level;
+ protected boolean started;
+
+ static {
+ NULL = new VoidPointable();
+ NULL.set(new byte[] { ATypeTag.NULL.serialize() }, 0, 1);
+
+ MISSING = new VoidPointable();
+ MISSING.set(new byte[] { ATypeTag.MISSING.serialize() }, 0, 1);
+ }
+
+ protected AbstractValueAssembler(int level, AssemblerInfo info) {
+ this.parent = info.getParent();
+ this.fieldName = info.getFieldName();
+ this.fieldIndex = info.getFieldIndex();
+ this.delegate = info.isDelegate();
+ this.level = level;
+ }
+
+ /**
+ * Add {@link ATypeTag#NULL} value to the ancestor at {@code nullLevel}
+ *
+ * @param nullLevel at what level the null occurred
+ */
+ abstract void addNullToAncestor(int nullLevel) throws HyracksDataException;
+
+ /**
+ * Add {@link ATypeTag#MISSING} value to the ancestor at {@code missingLevel}
+ *
+ * @param missingLevel at what level the missing occurred
+ */
+ abstract void addMissingToAncestor(int missingLevel) throws HyracksDataException;
+
+ /**
+ * Add the value of this assembler to its parent
+ */
+ abstract void addValueToParent() throws HyracksDataException;
+
+ /**
+ * @return the assembled value
+ */
+ public abstract IValueReference getValue() throws HyracksDataException;
+
+ /**
+ * Reset assembler
+ */
+ void reset() {
+ //NoOp
+ }
+
+ /**
+ * @return whether this assembler is the delegate (or representative) of its siblings
+ */
+ final boolean isDelegate() {
+ return delegate;
+ }
+
+ /**
+ * @return parent of the assembler
+ */
+ final AbstractNestedValueAssembler getParent() {
+ return parent;
+ }
+
+ /**
+ * Return the field name of the value of this assembler
+ */
+ final IValueReference getFieldName() {
+ return fieldName;
+ }
+
+ /**
+ * Return the field index of the value of this assembler (for closed types)
+ */
+ final int getFieldIndex() {
+ return fieldIndex;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayValueAssembler.java
new file mode 100644
index 0000000..2352e7f
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayValueAssembler.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.builders.IAsterixListBuilder;
+import org.apache.asterix.builders.ListBuilderFactory;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public class ArrayValueAssembler extends AbstractNestedValueAssembler {
+ private final IAsterixListBuilder listBuilder;
+ private final AbstractCollectionType collectionType;
+ private final int firstValueIndex;
+
+ ArrayValueAssembler(int level, AssemblerInfo info, int firstValueIndex) {
+ super(level, info);
+ this.firstValueIndex = firstValueIndex;
+ collectionType = (AbstractCollectionType) info.getDeclaredType();
+ listBuilder = new ListBuilderFactory().create(collectionType.getTypeTag());
+ }
+
+ final int getFirstValueIndex() {
+ return firstValueIndex;
+ }
+
+ @Override
+ void reset() {
+ listBuilder.reset(collectionType);
+ storage.reset();
+ }
+
+ @Override
+ void addValue(AbstractValueAssembler value) throws HyracksDataException {
+ listBuilder.addItem(value.getValue());
+ }
+
+ @Override
+ void addNull(AbstractValueAssembler value) throws HyracksDataException {
+ listBuilder.addItem(NULL);
+ }
+
+ @Override
+ void addMissing() throws HyracksDataException {
+ listBuilder.addItem(MISSING);
+ }
+
+ @Override
+ void addValueToParent() throws HyracksDataException {
+ storage.reset();
+ listBuilder.write(storage.getDataOutput(), true);
+ getParent().addValue(this);
+ }
+
+ @Override
+ public IValueReference getValue() {
+ return storage;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java
new file mode 100644
index 0000000..dcd240b
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class ArrayWithUnionValueAssembler extends ArrayValueAssembler {
+ private final int numberOfUnionChildren;
+ private int numberOfAddedValues;
+ private boolean nonMissingValueAdded;
+
+ ArrayWithUnionValueAssembler(int level, AssemblerInfo info, int firstValueIndex, AbstractSchemaNode itemNode) {
+ super(level, info, firstValueIndex);
+ this.numberOfUnionChildren = ((UnionSchemaNode) itemNode).getChildren().size();
+ }
+
+ @Override
+ void reset() {
+ numberOfAddedValues = 0;
+ nonMissingValueAdded = false;
+ super.reset();
+ }
+
+ @Override
+ void addValue(AbstractValueAssembler value) throws HyracksDataException {
+ nonMissingValueAdded = true;
+ numberOfAddedValues++;
+ super.addValue(value);
+ }
+
+ @Override
+ void addNull(AbstractValueAssembler value) throws HyracksDataException {
+ nonMissingValueAdded = true;
+ numberOfAddedValues++;
+ super.addNull(value);
+ }
+
+ @Override
+ void addMissing() throws HyracksDataException {
+ numberOfAddedValues++;
+ if (nonMissingValueAdded && numberOfAddedValues >= numberOfUnionChildren) {
+ nonMissingValueAdded = false;
+ numberOfAddedValues = numberOfAddedValues % numberOfUnionChildren;
+ } else if (numberOfAddedValues == numberOfUnionChildren) {
+ super.addMissing();
+ numberOfAddedValues = 0;
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java
new file mode 100644
index 0000000..969a091
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.asterix.column.assembler.value.IValueGetter;
+import org.apache.asterix.column.assembler.value.IValueGetterFactory;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ISchemaNodeVisitor;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.column.operation.query.QueryColumnMetadata;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+public class AssemblerBuilderVisitor implements ISchemaNodeVisitor<AbstractValueAssembler, AssemblerInfo> {
+ private static final BitSet NO_DECLARED_FIELDS = new BitSet(0);
+ private final QueryColumnMetadata columnMetadata;
+ private final IColumnValuesReaderFactory readerFactory;
+ private final List<AbstractPrimitiveValueAssembler> valueAssemblers;
+ private final IValueGetterFactory valueGetterFactory;
+ private AbstractValueAssembler rootAssembler;
+
+ //Recursion info
+ private final IntList delimiters;
+ private int level;
+
+ public AssemblerBuilderVisitor(QueryColumnMetadata columnMetadata, IColumnValuesReaderFactory readerFactory,
+ IValueGetterFactory valueGetterFactory) {
+ this.columnMetadata = columnMetadata;
+ this.readerFactory = readerFactory;
+ this.valueGetterFactory = valueGetterFactory;
+ valueAssemblers = new ArrayList<>();
+ delimiters = new IntArrayList();
+ }
+
+ public List<AbstractPrimitiveValueAssembler> createValueAssemblers(AbstractSchemaNode requestedSchema,
+ ARecordType declaredType) throws HyracksDataException {
+ EmptyAssembler root = new EmptyAssembler();
+ AssemblerInfo info = new AssemblerInfo(declaredType, root);
+ level = 0;
+ rootAssembler = requestedSchema.accept(this, info);
+ return valueAssemblers;
+ }
+
+ public AbstractValueAssembler getRootAssembler() {
+ return rootAssembler;
+ }
+
+ @Override
+ public AbstractValueAssembler visit(ObjectSchemaNode objectNode, AssemblerInfo info) throws HyracksDataException {
+ ObjectValueAssembler objectAssembler = new ObjectValueAssembler(level, info);
+ level++;
+
+ BitSet declaredFields = handleDeclaredFields(objectNode, info, objectAssembler);
+ IntList childrenFieldNameIndexes = objectNode.getChildrenFieldNameIndexes();
+ if (declaredFields.length() < childrenFieldNameIndexes.size()) {
+ //Open fields are requested
+ for (int i = 0; i < childrenFieldNameIndexes.size(); i++) {
+ int fieldNameIndex = childrenFieldNameIndexes.getInt(i);
+ AbstractSchemaNode childNode = objectNode.getChild(fieldNameIndex);
+ if (childNode.getTypeTag() != ATypeTag.MISSING && !declaredFields.get(fieldNameIndex)) {
+ IAType childType = getChildType(childNode, BuiltinType.ANY);
+ IValueReference fieldName = columnMetadata.getFieldNamesDictionary().getFieldName(fieldNameIndex);
+ //The last child should be a delegate
+ boolean delegate = i == childrenFieldNameIndexes.size() - 1;
+ AssemblerInfo childInfo = new AssemblerInfo(childType, objectAssembler, delegate, fieldName);
+ childNode.accept(this, childInfo);
+ }
+ }
+ }
+
+ level--;
+ return objectAssembler;
+ }
+
+ private BitSet handleDeclaredFields(ObjectSchemaNode objectNode, AssemblerInfo info,
+ ObjectValueAssembler objectAssembler) throws HyracksDataException {
+ ARecordType declaredType = (ARecordType) info.getDeclaredType();
+ if (declaredType == DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE) {
+ return NO_DECLARED_FIELDS;
+ }
+ BitSet processedFields = new BitSet();
+ String[] declaredFieldNames = declaredType.getFieldNames();
+ IAType[] declaredFieldTypes = declaredType.getFieldTypes();
+
+ // The last child of a declared field can be a delegate iff all requested fields are declared
+ boolean containsDelegate = objectNode.getChildren().size() == declaredFieldTypes.length;
+ for (int i = 0; i < declaredFieldTypes.length; i++) {
+ String fieldName = declaredFieldNames[i];
+ int fieldNameIndex = columnMetadata.getFieldNamesDictionary().getFieldNameIndex(fieldName);
+ //Check if the declared field was requested
+ AbstractSchemaNode childNode = objectNode.getChild(fieldNameIndex);
+ if (childNode.getTypeTag() != ATypeTag.MISSING) {
+ IAType childType = getChildType(childNode, declaredFieldTypes[i]);
+ processedFields.set(fieldNameIndex);
+ boolean delegate = containsDelegate && i == declaredFieldTypes.length - 1;
+ AssemblerInfo childInfo = new AssemblerInfo(childType, objectAssembler, delegate, i);
+ childNode.accept(this, childInfo);
+ }
+ }
+ return processedFields;
+ }
+
+ @Override
+ public AbstractValueAssembler visit(AbstractCollectionSchemaNode collectionNode, AssemblerInfo info)
+ throws HyracksDataException {
+ AbstractCollectionType declaredType = (AbstractCollectionType) info.getDeclaredType();
+ AbstractSchemaNode itemNode = collectionNode.getItemNode();
+
+ ArrayValueAssembler arrayAssembler = itemNode.getTypeTag() == ATypeTag.UNION
+ ? new ArrayWithUnionValueAssembler(level, info, valueAssemblers.size(), itemNode)
+ : new ArrayValueAssembler(level, info, valueAssemblers.size());
+ delimiters.add(level - 1);
+ level++;
+
+ IAType itemDeclaredType = getChildType(itemNode, declaredType.getItemType());
+ AssemblerInfo itemInfo = new AssemblerInfo(itemDeclaredType, arrayAssembler, false);
+ itemNode.accept(this, itemInfo);
+
+ //Add the array assembler to the last repeated value assembler
+ RepeatedPrimitiveValueAssembler repeatedAssembler =
+ (RepeatedPrimitiveValueAssembler) valueAssemblers.get(valueAssemblers.size() - 1);
+ repeatedAssembler.addArray(arrayAssembler);
+
+ level--;
+ delimiters.removeInt(delimiters.size() - 1);
+ return arrayAssembler;
+ }
+
+ @Override
+ public AbstractValueAssembler visit(UnionSchemaNode unionNode, AssemblerInfo info) throws HyracksDataException {
+ /*
+ * UnionSchemaNode does not actually exist. We know the parent of the union could have items of multiple types.
+ * Thus, the union's parent is the actual parent for all the union types
+ */
+ Collection<AbstractSchemaNode> children = unionNode.getChildren().values();
+ int index = 0;
+ for (AbstractSchemaNode node : children) {
+ IAType unionDeclaredType = getChildType(node, info.getDeclaredType());
+ boolean delegate = info.isDelegate() && index++ == children.size() - 1;
+ AssemblerInfo unionInfo = new AssemblerInfo(unionDeclaredType, info.getParent(), delegate,
+ info.getFieldName(), info.getFieldIndex(), true);
+ node.accept(this, unionInfo);
+ }
+ return info.getParent();
+ }
+
+ @Override
+ public AbstractValueAssembler visit(PrimitiveSchemaNode primitiveNode, AssemblerInfo info) {
+ AbstractPrimitiveValueAssembler assembler;
+ IValueGetter valueGetter = valueGetterFactory.createValueGetter(primitiveNode.getTypeTag());
+ if (!delimiters.isEmpty()) {
+ IColumnValuesReader reader = readerFactory.createValueReader(primitiveNode.getTypeTag(),
+ primitiveNode.getColumnIndex(), level, getDelimiters());
+ assembler = new RepeatedPrimitiveValueAssembler(level, info, reader, valueGetter);
+
+ } else {
+ IColumnValuesReader reader = readerFactory.createValueReader(primitiveNode.getTypeTag(),
+ primitiveNode.getColumnIndex(), level, primitiveNode.isPrimaryKey());
+ assembler = new PrimitiveValueAssembler(level, info, reader, valueGetter);
+ }
+ valueAssemblers.add(assembler);
+ return assembler;
+ }
+
+ private int[] getDelimiters() {
+ int numOfDelimiters = delimiters.size();
+ int[] reversed = new int[numOfDelimiters];
+ for (int i = 0; i < numOfDelimiters; i++) {
+ reversed[i] = delimiters.getInt(numOfDelimiters - i - 1);
+ }
+ return reversed;
+ }
+
+ private IAType getChildType(AbstractSchemaNode childNode, IAType childType) {
+ if (childType.getTypeTag() != ATypeTag.ANY) {
+ return childType;
+ }
+ ATypeTag childTypeTag = childNode.getTypeTag();
+ if (childTypeTag == ATypeTag.UNION) {
+ //Union type could return any type
+ return BuiltinType.ANY;
+ } else if (childTypeTag.isDerivedType()) {
+ return DefaultOpenFieldType.getDefaultOpenFieldType(childTypeTag);
+ } else {
+ return BuiltinType.getBuiltinType(childTypeTag);
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerInfo.java
new file mode 100644
index 0000000..712e65c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerInfo.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+public class AssemblerInfo {
+ private final AbstractNestedValueAssembler parent;
+ private final IAType declaredType;
+ private final boolean delegate;
+ private final IValueReference fieldName;
+ private final int fieldIndex;
+
+ public AssemblerInfo() {
+ this(BuiltinType.ANY, null, false);
+ }
+
+ public AssemblerInfo(IAType declaredType, EmptyAssembler parent) {
+ this(declaredType, parent, false);
+ }
+
+ public AssemblerInfo(IAType declaredType, AbstractNestedValueAssembler parent, boolean delegate) {
+ this(declaredType, parent, delegate, null, -1);
+ }
+
+ public AssemblerInfo(IAType declaredType, AbstractNestedValueAssembler parent, boolean delegate,
+ IValueReference fieldName) {
+ this(declaredType, parent, delegate, fieldName, -1);
+ }
+
+ public AssemblerInfo(IAType declaredType, AbstractNestedValueAssembler parent, boolean delegate, int fieldIndex) {
+ this(declaredType, parent, delegate, null, fieldIndex);
+ }
+
+ public AssemblerInfo(IAType declaredType, AbstractNestedValueAssembler parent, boolean delegate,
+ IValueReference fieldName, int fieldIndex) {
+ this(declaredType, parent, delegate, fieldName, fieldIndex, false);
+ }
+
+ public AssemblerInfo(IAType declaredType, AbstractNestedValueAssembler parent, boolean delegate,
+ IValueReference fieldName, int fieldIndex, boolean fieldNameTagged) {
+ this.parent = parent;
+ this.declaredType = declaredType;
+ this.delegate = delegate;
+ this.fieldName = fieldNameTagged ? fieldName : createTaggedFieldName(fieldName);
+ this.fieldIndex = fieldIndex;
+ }
+
+ private IValueReference createTaggedFieldName(IValueReference fieldName) {
+ if (fieldName == null) {
+ return null;
+ }
+ byte[] storage = new byte[1 + fieldName.getLength()];
+ storage[0] = ATypeTag.STRING.serialize();
+ System.arraycopy(fieldName.getByteArray(), fieldName.getStartOffset(), storage, 1, fieldName.getLength());
+ VoidPointable taggedFieldName = new VoidPointable();
+ taggedFieldName.set(storage, 0, storage.length);
+ return taggedFieldName;
+ }
+
+ public AbstractNestedValueAssembler getParent() {
+ return parent;
+ }
+
+ public IAType getDeclaredType() {
+ return declaredType;
+ }
+
+ public boolean isDelegate() {
+ return delegate;
+ }
+
+ public IValueReference getFieldName() {
+ return fieldName;
+ }
+
+ public int getFieldIndex() {
+ return fieldIndex;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/EmptyAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/EmptyAssembler.java
new file mode 100644
index 0000000..406a401
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/EmptyAssembler.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public class EmptyAssembler extends AbstractNestedValueAssembler {
+
+ EmptyAssembler() {
+ super(-1, new AssemblerInfo());
+ }
+
+ @Override
+ void addValue(AbstractValueAssembler value) throws HyracksDataException {
+ //noOp
+ }
+
+ @Override
+ void addValueToParent() throws HyracksDataException {
+ //noOp
+ }
+
+ @Override
+ void addNull(AbstractValueAssembler value) throws HyracksDataException {
+ //noOp
+ }
+
+ @Override
+ public IValueReference getValue() throws HyracksDataException {
+ return null;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java
new file mode 100644
index 0000000..536ce02
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public class ObjectValueAssembler extends AbstractNestedValueAssembler {
+ private final RecordBuilder recordBuilder;
+ private final ARecordType recordType;
+
+ ObjectValueAssembler(int level, AssemblerInfo info) {
+ super(level, info);
+ recordBuilder = new RecordBuilder();
+ recordType = (ARecordType) info.getDeclaredType();
+ }
+
+ @Override
+ void reset() {
+ recordBuilder.reset(recordType);
+ storage.reset();
+ }
+
+ @Override
+ void addValue(AbstractValueAssembler value) throws HyracksDataException {
+ int valueIndex = value.getFieldIndex();
+ if (valueIndex >= 0) {
+ recordBuilder.addField(valueIndex, value.getValue());
+ } else {
+ recordBuilder.addField(value.getFieldName(), value.getValue());
+ }
+ }
+
+ @Override
+ void addNull(AbstractValueAssembler value) throws HyracksDataException {
+ int valueIndex = value.getFieldIndex();
+ if (valueIndex >= 0) {
+ recordBuilder.addField(valueIndex, NULL);
+ } else {
+ recordBuilder.addField(value.getFieldName(), NULL);
+ }
+ }
+
+ @Override
+ void addValueToParent() throws HyracksDataException {
+ storage.reset();
+ recordBuilder.write(storage.getDataOutput(), true);
+ getParent().addValue(this);
+ }
+
+ @Override
+ public IValueReference getValue() {
+ return storage;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/PrimitiveValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/PrimitiveValueAssembler.java
new file mode 100644
index 0000000..9592a12
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/PrimitiveValueAssembler.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import org.apache.asterix.column.assembler.value.IValueGetter;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class PrimitiveValueAssembler extends AbstractPrimitiveValueAssembler {
+
+ PrimitiveValueAssembler(int level, AssemblerInfo info, IColumnValuesReader reader, IValueGetter primitiveValue) {
+ super(level, info, reader, primitiveValue);
+ }
+
+ @Override
+ public int next() throws HyracksDataException {
+ if (!reader.next()) {
+ throw new IllegalAccessError("no more values");
+ } else if (reader.isNull() && (isDelegate() || reader.getLevel() + 1 == level)) {
+ addNullToAncestor(reader.getLevel());
+ } else if (reader.isValue()) {
+ addValueToParent();
+ }
+
+ if (isDelegate()) {
+ getParent().end();
+ }
+ //Go to next value
+ return -1;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java
new file mode 100644
index 0000000..8fa228f
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.column.assembler.value.IValueGetter;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+class RepeatedPrimitiveValueAssembler extends AbstractPrimitiveValueAssembler {
+ private final List<ArrayValueAssembler> arrays;
+
+ RepeatedPrimitiveValueAssembler(int level, AssemblerInfo info, IColumnValuesReader reader,
+ IValueGetter primitiveValue) {
+ super(level, info, reader, primitiveValue);
+ this.arrays = new ArrayList<>();
+ }
+
+ public void addArray(ArrayValueAssembler assembler) {
+ arrays.add(assembler);
+ }
+
+ @Override
+ public int next() throws HyracksDataException {
+ if (!reader.next()) {
+ throw new IllegalStateException("No more values");
+ } else if (reader.isNull() && (!arrays.isEmpty() || reader.getLevel() + 1 == level)) {
+ /*
+ * There are two cases here for where the null belongs to:
+ * 1- If the null is an array item, then add it
+ * 2- If the null is an ancestor, then we only add null if this column is the array delegate
+ * (i.e., !arrays.isEmpty())
+ */
+ addNullToAncestor(reader.getLevel());
+ } else if (reader.isMissing() && reader.getLevel() + 1 == level) {
+ /*
+ * Add a missing item
+ */
+ addMissingToAncestor(reader.getLevel());
+ } else if (reader.isValue()) {
+ addValueToParent();
+ }
+
+ if (isDelegate()) {
+ getParent().end();
+ }
+
+ //Initially, go to the next primitive assembler
+ int nextIndex = NEXT_ASSEMBLER;
+ if (!arrays.isEmpty()) {
+ /*
+ * This assembler is a delegate of a repeated group
+ * The delimiter index tells us that this assembler is responsible for a finished group
+ */
+ int delimiterIndex = reader.getDelimiterIndex();
+ if (delimiterIndex < arrays.size() && reader.isDelimiter()) {
+ //Also finish the next group
+ delimiterIndex++;
+ }
+
+ int numberOfFinishedGroups = Math.min(delimiterIndex, arrays.size());
+ for (int i = 0; i < numberOfFinishedGroups; i++) {
+ //I'm the delegate for this group of repeated values and the group(s) is finished
+ ArrayValueAssembler assembler = arrays.get(i);
+ assembler.end();
+ }
+
+ //Is the repeated group (determined by the delimiter index) still unfinished?
+ if (delimiterIndex < arrays.size()) {
+ //Yes, go to the first value of the unfinished repeated group
+ nextIndex = arrays.get(delimiterIndex).getFirstValueIndex();
+ }
+ }
+
+ //Go to next value
+ return nextIndex;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/AbstractFixedLengthValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/AbstractFixedLengthValueGetter.java
new file mode 100644
index 0000000..aeef686
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/AbstractFixedLengthValueGetter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+public abstract class AbstractFixedLengthValueGetter implements IValueGetter {
+ protected final VoidPointable value;
+
+ AbstractFixedLengthValueGetter(ATypeTag typeTag, int nonTaggedLength) {
+ //+1 for the type tag
+ byte[] storage = new byte[1 + nonTaggedLength];
+ storage[0] = typeTag.serialize();
+ value = new VoidPointable();
+ value.set(storage, 0, storage.length);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/BooleanValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/BooleanValueGetter.java
new file mode 100644
index 0000000..4a776ab
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/BooleanValueGetter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.BooleanPointable;
+
+class BooleanValueGetter extends AbstractFixedLengthValueGetter {
+ BooleanValueGetter() {
+ super(ATypeTag.BOOLEAN, 1);
+ }
+
+ @Override
+ public IValueReference getValue(IColumnValuesReader reader) {
+ BooleanPointable.setBoolean(value.getByteArray(), value.getStartOffset() + 1, reader.getBoolean());
+ return value;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/DoubleValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/DoubleValueGetter.java
new file mode 100644
index 0000000..2e88896
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/DoubleValueGetter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+
+class DoubleValueGetter extends AbstractFixedLengthValueGetter {
+ DoubleValueGetter() {
+ super(ATypeTag.DOUBLE, Double.BYTES);
+ }
+
+ @Override
+ public IValueReference getValue(IColumnValuesReader reader) {
+ DoublePointable.setDouble(value.getByteArray(), value.getStartOffset() + 1, reader.getDouble());
+ return value;
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/IValueGetter.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/IValueGetter.java
index 3c1a24d..9e58ab8 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/IValueGetter.java
@@ -16,20 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.assembler.value;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.hyracks.data.std.api.IValueReference;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+@FunctionalInterface
+public interface IValueGetter {
+ IValueReference getValue(IColumnValuesReader reader);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/IValueGetterFactory.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/IValueGetterFactory.java
index 3c1a24d..0b58cfc4 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/IValueGetterFactory.java
@@ -16,20 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.assembler.value;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import org.apache.asterix.om.types.ATypeTag;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+@FunctionalInterface
+public interface IValueGetterFactory {
+ IValueGetter createValueGetter(ATypeTag typeTag);
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/LongValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/LongValueGetter.java
new file mode 100644
index 0000000..e76e3c9
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/LongValueGetter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+
+class LongValueGetter extends AbstractFixedLengthValueGetter {
+ LongValueGetter() {
+ super(ATypeTag.BIGINT, Long.BYTES);
+ }
+
+ @Override
+ public IValueReference getValue(IColumnValuesReader reader) {
+ LongPointable.setLong(value.getByteArray(), value.getStartOffset() + 1, reader.getLong());
+ return value;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/MissingValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/MissingValueGetter.java
new file mode 100644
index 0000000..1ae84ee
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/MissingValueGetter.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+public class MissingValueGetter implements IValueGetter {
+ public static final IValueGetter INSTANCE = new MissingValueGetter();
+ private static final VoidPointable MISSING;
+
+ static {
+ MISSING = new VoidPointable();
+ MISSING.set(new byte[] { ATypeTag.MISSING.serialize() }, 0, 1);
+ }
+
+ private MissingValueGetter() {
+ }
+
+ @Override
+ public IValueReference getValue(IColumnValuesReader reader) {
+ return MISSING;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/NullValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/NullValueGetter.java
new file mode 100644
index 0000000..e050252
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/NullValueGetter.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+public class NullValueGetter implements IValueGetter {
+ public static final IValueGetter INSTANCE = new NullValueGetter();
+ private static final VoidPointable NULL;
+
+ static {
+ NULL = new VoidPointable();
+ NULL.set(new byte[] { ATypeTag.NULL.serialize() }, 0, 1);
+ }
+
+ private NullValueGetter() {
+ }
+
+ @Override
+ public IValueReference getValue(IColumnValuesReader reader) {
+ return NULL;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/StringValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/StringValueGetter.java
new file mode 100644
index 0000000..1dd1aa7
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/StringValueGetter.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+class StringValueGetter implements IValueGetter {
+ private final ArrayBackedValueStorage value;
+
+ public StringValueGetter() {
+ value = new ArrayBackedValueStorage();
+ }
+
+ @Override
+ public IValueReference getValue(IColumnValuesReader reader) {
+ IValueReference string = reader.getBytes();
+ value.setSize(1 + string.getLength());
+ byte[] bytes = value.getByteArray();
+ bytes[0] = ATypeTag.STRING.serialize();
+ System.arraycopy(string.getByteArray(), string.getStartOffset(), bytes, 1, string.getLength());
+ return value;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/UUIDValueGetter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/UUIDValueGetter.java
new file mode 100644
index 0000000..135ed85
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/UUIDValueGetter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+class UUIDValueGetter extends AbstractFixedLengthValueGetter {
+ UUIDValueGetter() {
+ super(ATypeTag.UUID, 16);
+ }
+
+ @Override
+ public IValueReference getValue(IColumnValuesReader reader) {
+ IValueReference uuid = reader.getBytes();
+ System.arraycopy(uuid.getByteArray(), uuid.getStartOffset(), value.getByteArray(), value.getStartOffset() + 1,
+ uuid.getLength());
+ return value;
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/ValueGetterFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/ValueGetterFactory.java
new file mode 100644
index 0000000..5f7fd7e
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/value/ValueGetterFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.assembler.value;
+
+import org.apache.asterix.om.types.ATypeTag;
+
+public class ValueGetterFactory implements IValueGetterFactory {
+ public static final IValueGetterFactory INSTANCE = new ValueGetterFactory();
+
+ private ValueGetterFactory() {
+ }
+
+ @Override
+ public IValueGetter createValueGetter(ATypeTag typeTag) {
+ switch (typeTag) {
+ case NULL:
+ return NullValueGetter.INSTANCE;
+ case MISSING:
+ return MissingValueGetter.INSTANCE;
+ case BOOLEAN:
+ return new BooleanValueGetter();
+ case BIGINT:
+ return new LongValueGetter();
+ case DOUBLE:
+ return new DoubleValueGetter();
+ case STRING:
+ return new StringValueGetter();
+ case UUID:
+ return new UUIDValueGetter();
+ default:
+ throw new UnsupportedOperationException(typeTag + " is not supported");
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/ParquetDeltaBinaryPackingConfig.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/ParquetDeltaBinaryPackingConfig.java
new file mode 100644
index 0000000..f591d57
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/ParquetDeltaBinaryPackingConfig.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+
+/**
+ * Copy of {@link org.apache.parquet.column.values.delta.DeltaBinaryPackingConfig}
+ */
+public class ParquetDeltaBinaryPackingConfig {
+ private int blockSizeInValues;
+ private int miniBlockNumInABlock;
+ private int miniBlockSizeInValues;
+
+ public ParquetDeltaBinaryPackingConfig(int blockSizeInValues, int miniBlockNumInABlock) {
+ reset(blockSizeInValues, miniBlockNumInABlock);
+ }
+
+ private void reset(int blockSizeInValues, int miniBlockNumInABlock) {
+ this.blockSizeInValues = blockSizeInValues;
+ this.miniBlockNumInABlock = miniBlockNumInABlock;
+ double miniSize = (double) blockSizeInValues / miniBlockNumInABlock;
+ Preconditions.checkArgument(miniSize % 8 == 0, "miniBlockSize must be multiple of 8, but it's " + miniSize);
+ this.miniBlockSizeInValues = (int) miniSize;
+ }
+
+ public static ParquetDeltaBinaryPackingConfig readConfig(InputStream in, ParquetDeltaBinaryPackingConfig config)
+ throws IOException {
+ final int blockSizeInValues = BytesUtils.readUnsignedVarInt(in);
+ final int miniBlockNumInABlock = BytesUtils.readUnsignedVarInt(in);
+ if (config == null) {
+ return new ParquetDeltaBinaryPackingConfig(blockSizeInValues, miniBlockNumInABlock);
+ }
+ config.reset(blockSizeInValues, miniBlockNumInABlock);
+ return config;
+ }
+
+ public BytesInput toBytesInput() {
+ return BytesInput.concat(BytesInput.fromUnsignedVarInt(blockSizeInValues),
+ BytesInput.fromUnsignedVarInt(miniBlockNumInABlock));
+ }
+
+ public int getBlockSizeInValues() {
+ return blockSizeInValues;
+ }
+
+ public int getMiniBlockNumInABlock() {
+ return miniBlockNumInABlock;
+ }
+
+ public int getMiniBlockSizeInValues() {
+ return miniBlockSizeInValues;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/AbstractParquetValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/AbstractParquetValuesReader.java
new file mode 100644
index 0000000..5f5b88c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/AbstractParquetValuesReader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.decoder;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.column.values.ValuesReader;
+
+/**
+ * Replaces {@link ValuesReader}
+ */
+public abstract class AbstractParquetValuesReader {
+ public abstract void initFromPage(AbstractBytesInputStream stream) throws IOException;
+
+ public abstract void skip();
+
+ public int readInteger() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public long readLong() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public double readDouble() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public IValueReference readBytes() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaBinaryPackingValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaBinaryPackingValuesReader.java
new file mode 100644
index 0000000..9aafa0f
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaBinaryPackingValuesReader.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.decoder;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.asterix.column.bytes.ParquetDeltaBinaryPackingConfig;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/**
+ * Re-implementation of {@link DeltaBinaryPackingValuesReader}
+ */
+public class ParquetDeltaBinaryPackingValuesReader extends AbstractParquetValuesReader {
+ private int totalValueCount;
+ /**
+ * values read by the caller
+ */
+ private int valuesBufferedRead;
+ private int valuesRead;
+
+ /**
+ * stores the decoded values including the first value which is written to the header
+ */
+ private long[] valuesBuffer;
+ /**
+ * values loaded to the buffer, it could be bigger than the totalValueCount
+ * when data is not aligned to mini block, which means padding 0s are in the buffer
+ */
+ private int valuesBuffered;
+ private AbstractBytesInputStream in;
+ private ParquetDeltaBinaryPackingConfig config;
+ private int[] bitWidths;
+ private ByteBuffer bitWidthBuffer;
+ private long lastElement;
+
+ /**
+ * Loads one block at a time instead of eagerly loading all blocks in {@link DeltaBinaryPackingValuesReader}.
+ * This is to fix the {@link #valuesBuffer} size
+ */
+ @Override
+ public void initFromPage(AbstractBytesInputStream stream) throws IOException {
+ this.in = stream;
+ this.config = ParquetDeltaBinaryPackingConfig.readConfig(in, this.config);
+ this.totalValueCount = BytesUtils.readUnsignedVarInt(in);
+ allocateValuesBuffer();
+ bitWidths = allocate(bitWidths, config.getMiniBlockNumInABlock());
+ valuesBuffered = 0;
+
+ valuesBufferedRead = 0;
+ valuesRead = 0;
+
+ //read first value from header
+ valuesBuffer[valuesBuffered++] = BytesUtils.readZigZagVarLong(in);
+ lastElement = valuesBuffer[0];
+
+ if (valuesBuffered < totalValueCount) {
+ loadNewBlockToBuffer();
+ }
+ }
+
+ /**
+ * the value buffer is allocated so that the size of it is multiple of mini block
+ * because when writing, data is flushed on a mini block basis
+ */
+ private void allocateValuesBuffer() {
+ //+ 1 because first value written to header is also stored in values buffer
+ final int bufferSize = config.getMiniBlockSizeInValues() * config.getMiniBlockNumInABlock() + 1;
+ if (valuesBuffer == null || valuesBuffer.length < bufferSize) {
+ valuesBuffer = new long[bufferSize];
+ } else {
+ Arrays.fill(valuesBuffer, 0);
+ }
+ }
+
+ private int[] allocate(int[] array, int size) {
+ if (array == null || array.length < size) {
+ return new int[size];
+ }
+ return array;
+ }
+
+ @Override
+ public void skip() {
+ checkRead();
+ valuesRead++;
+ }
+
+ @Override
+ public int readInteger() {
+ // TODO: probably implement it separately
+ return (int) readLong();
+ }
+
+ @Override
+ public long readLong() {
+ checkRead();
+ valuesRead++;
+ return valuesBuffer[valuesBufferedRead++];
+ }
+
+ private void checkRead() {
+ if (valuesRead >= totalValueCount) {
+ throw new ParquetDecodingException("no more value to read, total value count is " + totalValueCount);
+ }
+ if (valuesBufferedRead >= valuesBuffered) {
+ //Set the last value buffered as the first
+ lastElement = valuesBuffer[valuesBufferedRead - 1];
+ valuesBufferedRead = 0;
+ valuesBuffered = 0;
+ Arrays.fill(valuesBuffer, 0);
+ try {
+ loadNewBlockToBuffer();
+ } catch (IOException e) {
+ throw new ParquetDecodingException("can not load next block", e);
+ }
+
+ }
+ }
+
+ private void loadNewBlockToBuffer() throws IOException {
+ long minDeltaInCurrentBlock;
+ try {
+ minDeltaInCurrentBlock = BytesUtils.readZigZagVarLong(in);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("can not read min delta in current block", e);
+ }
+
+ readBitWidthsForMiniBlocks();
+
+ // mini block is atomic for reading, we read a mini block when there are more values left
+ int i;
+ for (i = 0; i < config.getMiniBlockNumInABlock() && valuesRead + valuesBuffered < totalValueCount; i++) {
+ BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidths[i]);
+ unpackMiniBlock(packer);
+ }
+
+ //calculate values from deltas unpacked for current block
+ int valueUnpacked = i * config.getMiniBlockSizeInValues();
+ long prev = lastElement;
+ for (int j = valuesBuffered - valueUnpacked; j < valuesBuffered; j++) {
+ valuesBuffer[j] += minDeltaInCurrentBlock + prev;
+ prev = valuesBuffer[j];
+ }
+ }
+
+ /**
+ * mini block has a size of 8*n, unpack 8 value each time
+ *
+ * @param packer the packer created from bitwidth of current mini block
+ */
+ private void unpackMiniBlock(BytePackerForLong packer) throws IOException {
+ for (int j = 0; j < config.getMiniBlockSizeInValues(); j += 8) {
+ unpack8Values(packer);
+ }
+ }
+
+ private void unpack8Values(BytePackerForLong packer) throws IOException {
+ // get a single buffer of 8 values. most of the time, this won't require a copy
+ ByteBuffer buffer = readBitWidth(packer.getBitWidth());
+ packer.unpack8Values(buffer, buffer.position(), valuesBuffer, valuesBuffered);
+ this.valuesBuffered += 8;
+ }
+
+ private void readBitWidthsForMiniBlocks() {
+ for (int i = 0; i < config.getMiniBlockNumInABlock(); i++) {
+ try {
+ bitWidths[i] = BytesUtils.readIntLittleEndianOnOneByte(in);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("Can not decode bit width in block header", e);
+ }
+ }
+ }
+
+ private ByteBuffer prepareBitWidthBuffer(int length) {
+ if (bitWidthBuffer == null || bitWidthBuffer.capacity() < length) {
+ bitWidthBuffer = ByteBuffer.allocate(length);
+ }
+ bitWidthBuffer.clear();
+ bitWidthBuffer.limit(length);
+ return bitWidthBuffer;
+ }
+
+ private ByteBuffer readBitWidth(int length) throws IOException {
+ ByteBuffer buffer = prepareBitWidthBuffer(length);
+ int read = in.read(buffer);
+ if (read != length) {
+ throw new EOFException("Reached end of stream");
+ }
+ buffer.position(0);
+ return buffer;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaByteArrayReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaByteArrayReader.java
new file mode 100644
index 0000000..70c25b8
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaByteArrayReader.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.decoder;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.deltastrings.DeltaByteArrayReader;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/**
+ * Re-implementation of {@link DeltaByteArrayReader}
+ */
+public class ParquetDeltaByteArrayReader extends AbstractParquetValuesReader {
+ private final AbstractParquetValuesReader prefixLengthReader;
+ private final ParquetDeltaLengthByteArrayValuesReader suffixReader;
+ private final byte[] lengthBytes;
+
+ private final ArrayBackedValueStorage temp;
+ private final ArrayBackedValueStorage previous;
+ boolean newPage;
+
+ public ParquetDeltaByteArrayReader(boolean containsLength) {
+ this.prefixLengthReader = new ParquetDeltaBinaryPackingValuesReader();
+ this.suffixReader = new ParquetDeltaLengthByteArrayValuesReader();
+ this.temp = new ArrayBackedValueStorage();
+ this.previous = new ArrayBackedValueStorage();
+ lengthBytes = containsLength ? new byte[4] : new byte[0];
+ }
+
+ @Override
+ public void initFromPage(AbstractBytesInputStream stream) throws IOException {
+ AbstractBytesInputStream prefixStream = stream.sliceStream(BytesUtils.readUnsignedVarInt(stream));
+ prefixLengthReader.initFromPage(prefixStream);
+ suffixReader.initFromPage(stream);
+ previous.reset();
+ temp.reset();
+ newPage = true;
+ }
+
+ @Override
+ public void skip() {
+ // read the next value to skip so that previous is correct.
+ this.readBytes();
+ }
+
+ @Override
+ public IValueReference readBytes() {
+ int prefixLength = prefixLengthReader.readInteger();
+ // This does not copy bytes
+ IValueReference suffix = suffixReader.readBytes();
+
+ // NOTE: due to PARQUET-246, it is important that we
+ // respect prefixLength which was read from prefixLengthReader,
+ // even for the *first* value of a page. Even though the first
+ // value of the page should have an empty prefix, it may not
+ // because of PARQUET-246.
+
+ // We have to do this to materialize the output
+ try {
+ int lengthSize;
+ if (prefixLength != 0) {
+ lengthSize = appendLength(prefixLength + suffix.getLength());
+ temp.append(previous.getByteArray(), previous.getStartOffset(), prefixLength);
+ } else {
+ lengthSize = appendLength(suffix.getLength());
+ }
+ temp.append(suffix);
+ /*
+ * Adding length after appending prefix and suffix is important as we do not overwrite the original
+ * previous bytes
+ * */
+ System.arraycopy(lengthBytes, 0, temp.getByteArray(), 0, lengthSize);
+ previous.set(temp.getByteArray(), temp.getStartOffset() + lengthSize, temp.getLength() - lengthSize);
+ } catch (IOException e) {
+ throw new ParquetDecodingException(e);
+ }
+ newPage = false;
+ return temp;
+ }
+
+ private int appendLength(int length) {
+ if (lengthBytes.length > 0) {
+ int numOfBytes = UTF8StringUtil.encodeUTF8Length(length, lengthBytes, 0);
+ temp.setSize(numOfBytes);
+ return numOfBytes;
+ }
+ temp.setSize(0);
+ return 0;
+ }
+
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaLengthByteArrayValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaLengthByteArrayValuesReader.java
new file mode 100644
index 0000000..9913269
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDeltaLengthByteArrayValuesReader.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.decoder;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.io.ParquetDecodingException;
+
+public class ParquetDeltaLengthByteArrayValuesReader extends AbstractParquetValuesReader {
+
+ private final VoidPointable value;
+ private final AbstractParquetValuesReader lengthReader;
+ private AbstractBytesInputStream in;
+
+ public ParquetDeltaLengthByteArrayValuesReader() {
+ this.lengthReader = new ParquetDeltaBinaryPackingValuesReader();
+ value = new VoidPointable();
+ }
+
+ @Override
+ public void initFromPage(AbstractBytesInputStream stream) throws IOException {
+ AbstractBytesInputStream lengthStream = stream.sliceStream(BytesUtils.readUnsignedVarInt(stream));
+ lengthReader.initFromPage(lengthStream);
+ this.in = stream;
+ }
+
+ @Override
+ public void skip() {
+ int length = lengthReader.readInteger();
+ try {
+ in.skipFully(length);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("Failed to skip " + length + " bytes");
+ }
+ }
+
+ @Override
+ public IValueReference readBytes() {
+ int length = lengthReader.readInteger();
+ try {
+ in.read(value, length);
+ return value;
+ } catch (IOException e) {
+ throw new ParquetDecodingException("Failed to read " + length + " bytes");
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDoublePlainValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDoublePlainValuesReader.java
new file mode 100644
index 0000000..196bec2
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetDoublePlainValuesReader.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.decoder;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.parquet.bytes.LittleEndianDataInputStream;
+import org.apache.parquet.io.ParquetDecodingException;
+
+public class ParquetDoublePlainValuesReader extends AbstractParquetValuesReader {
+ private LittleEndianDataInputStream in;
+
+ @Override
+ public void initFromPage(AbstractBytesInputStream stream) throws IOException {
+ this.in = new LittleEndianDataInputStream(stream.remainingStream());
+ }
+
+ @Override
+ public void skip() {
+ try {
+ in.skipBytes(8);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("could not skip double", e);
+ }
+ }
+
+ @Override
+ public double readDouble() {
+ try {
+ return in.readDouble();
+ } catch (IOException e) {
+ throw new ParquetDecodingException("could not read double", e);
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java
new file mode 100644
index 0000000..4607dc2
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.decoder;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/**
+ * Re-implementation of {@link RunLengthBitPackingHybridDecoder}
+ */
+public class ParquetRunLengthBitPackingHybridDecoder {
+ private enum MODE {
+ RLE,
+ PACKED
+ }
+
+ private final int bitWidth;
+ private final BytePacker packer;
+ private InputStream in;
+
+ private MODE mode;
+ private int currentCount;
+ private int currentValue;
+ private int currentBufferLength;
+ private int[] currentBuffer;
+ private byte[] bytes;
+
+ public ParquetRunLengthBitPackingHybridDecoder(int bitWidth) {
+ Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");
+ this.bitWidth = bitWidth;
+ this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ }
+
+ public void reset(InputStream in) {
+ this.in = in;
+ currentCount = 0;
+ currentBufferLength = 0;
+ }
+
+ public int readInt() throws HyracksDataException {
+ try {
+ return nextInt();
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private int nextInt() throws IOException {
+ if (currentCount == 0) {
+ readNext();
+ }
+ --currentCount;
+ int result;
+ switch (mode) {
+ case RLE:
+ result = currentValue;
+ break;
+ case PACKED:
+ result = currentBuffer[currentBufferLength - 1 - currentCount];
+ break;
+ default:
+ throw new ParquetDecodingException("not a valid mode " + mode);
+ }
+ return result;
+ }
+
+ private void readNext() throws IOException {
+ Preconditions.checkArgument(in.available() > 0, "Reading past RLE/BitPacking stream.");
+ final int header = BytesUtils.readUnsignedVarInt(in);
+ mode = (header & 1) == 0 ? MODE.RLE : MODE.PACKED;
+ switch (mode) {
+ case RLE:
+ currentCount = header >>> 1;
+ currentValue = BytesUtils.readIntLittleEndianPaddedOnBitWidth(in, bitWidth);
+ break;
+ case PACKED:
+ int numGroups = header >>> 1;
+ currentCount = numGroups * 8;
+ allocateBuffers(currentCount, numGroups * bitWidth);
+ // At the end of the file RLE data though, there might not be that many bytes left.
+ int bytesToRead = (int) Math.ceil(currentCount * bitWidth / 8.0);
+ bytesToRead = Math.min(bytesToRead, in.available());
+ readFully(bytes, bytesToRead);
+ for (int valueIndex = 0, byteIndex = 0; valueIndex < currentCount; valueIndex += 8, byteIndex +=
+ bitWidth) {
+ packer.unpack8Values(bytes, byteIndex, currentBuffer, valueIndex);
+ }
+ break;
+ default:
+ throw new ParquetDecodingException("not a valid mode " + mode);
+ }
+ }
+
+ private void allocateBuffers(int intBufferLength, int byteBufferLength) {
+ if (currentBuffer == null || currentBuffer.length < intBufferLength) {
+ currentBuffer = new int[intBufferLength];
+ } else {
+ Arrays.fill(currentBuffer, 0);
+ }
+ currentBufferLength = intBufferLength;
+
+ if (bytes == null || bytes.length < byteBufferLength) {
+ bytes = new byte[byteBufferLength];
+ } else {
+ Arrays.fill(bytes, (byte) 0);
+ }
+ }
+
+ private void readFully(byte[] b, int len) throws IOException {
+ if (len < 0)
+ throw new IndexOutOfBoundsException();
+ int n = 0;
+ while (n < len) {
+ int count = in.read(b, n, len - n);
+ if (count < 0)
+ throw new EOFException();
+ n += count;
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/AbstractParquetDeltaBinaryPackingValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/AbstractParquetDeltaBinaryPackingValuesWriter.java
new file mode 100644
index 0000000..3102063
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/AbstractParquetDeltaBinaryPackingValuesWriter.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.ParquetDeltaBinaryPackingConfig;
+import org.apache.asterix.column.bytes.stream.out.MultiTemporaryBufferBytesOutputStream;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter;
+import org.apache.parquet.io.ParquetEncodingException;
+
+/**
+ * Re-implementation of {@link DeltaBinaryPackingValuesWriter}
+ */
+public abstract class AbstractParquetDeltaBinaryPackingValuesWriter extends AbstractParquetValuesWriter {
+
+ public static final int DEFAULT_NUM_BLOCK_VALUES = 128;
+
+ public static final int DEFAULT_NUM_MINIBLOCKS = 4;
+
+ protected final MultiTemporaryBufferBytesOutputStream outputStream;
+
+ /**
+ * stores blockSizeInValues, miniBlockNumInABlock and miniBlockSizeInValues
+ */
+ protected final ParquetDeltaBinaryPackingConfig config;
+
+ /**
+ * bit width for each mini block, reused between flushes
+ */
+ protected final int[] bitWidths;
+
+ protected int totalValueCount = 0;
+
+ /**
+ * a pointer to deltaBlockBuffer indicating the end of deltaBlockBuffer
+ * the number of values in the deltaBlockBuffer that haven't flushed to baos
+ * it will be reset after each flush
+ */
+ protected int deltaValuesToFlush = 0;
+
+ /**
+ * bytes buffer for a mini block, it is reused for each mini block.
+ * Therefore the size of biggest miniblock with bitwith of MAX_BITWITH is allocated
+ */
+ protected byte[] miniBlockByteBuffer;
+
+ /**
+ * Estimated element size after encoding
+ */
+ protected int estimatedElementSize = 0;
+ /**
+ * Estimated size for all non-flushed elements
+ */
+ protected int estimatedSize = 0;
+
+ protected AbstractParquetDeltaBinaryPackingValuesWriter(int blockSizeInValues, int miniBlockNum,
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ this.config = new ParquetDeltaBinaryPackingConfig(blockSizeInValues, miniBlockNum);
+ bitWidths = new int[config.getMiniBlockNumInABlock()];
+ outputStream = new MultiTemporaryBufferBytesOutputStream(multiPageOpRef);
+ }
+
+ protected void writeBitWidthForMiniBlock(int i) {
+ try {
+ BytesUtils.writeIntLittleEndianOnOneByte(outputStream, bitWidths[i]);
+ } catch (IOException e) {
+ throw new ParquetEncodingException("can not write bit width for mini-block", e);
+ }
+ }
+
+ protected int getMiniBlockCountToFlush(double numberCount) {
+ return (int) Math.ceil(numberCount / config.getMiniBlockSizeInValues());
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ this.totalValueCount = 0;
+ this.outputStream.reset();
+ this.deltaValuesToFlush = 0;
+ }
+
+ @Override
+ public void close() {
+ this.totalValueCount = 0;
+ this.deltaValuesToFlush = 0;
+ outputStream.finish();
+ }
+
+ @Override
+ public int getEstimatedSize() {
+ return outputStream.size() + estimatedSize;
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return outputStream.capacity();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/AbstractParquetValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/AbstractParquetValuesWriter.java
new file mode 100644
index 0000000..b53ded2
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/AbstractParquetValuesWriter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.column.values.ValuesWriter;
+
+/**
+ * Replaces {@link ValuesWriter}
+ */
+public abstract class AbstractParquetValuesWriter {
+
+ public abstract BytesInput getBytes();
+
+ /**
+ * called after getBytes() to reset the current buffer and start writing the next page
+ */
+ public abstract void reset() throws HyracksDataException;
+
+ /**
+ * Called to close the values writer. Any output stream is closed and can no longer be used.
+ * All resources are released.
+ */
+ public abstract void close();
+
+ public abstract int getEstimatedSize();
+
+ /**
+ * @return the allocated size of the buffer
+ */
+ public abstract int getAllocatedSize();
+
+ /**
+ * @param v the value to encode
+ */
+ public void writeBoolean(boolean v) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ /**
+ * @param v the value to encode
+ * @param skipLengthBytes whether to skip the length bytes of {@link UTF8StringPointable} or not
+ */
+ public void writeBytes(IValueReference v, boolean skipLengthBytes) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ /**
+ * @param v the value to encode
+ */
+ public void writeInteger(int v) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ /**
+ * @param v the value to encode
+ */
+ public void writeLong(long v) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ /**
+ * @param v the value to encode
+ */
+ public void writeDouble(double v) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaBinaryPackingValuesWriterForInteger.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaBinaryPackingValuesWriterForInteger.java
new file mode 100644
index 0000000..1c474fc
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaBinaryPackingValuesWriterForInteger.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import java.io.IOException;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger;
+import org.apache.parquet.io.ParquetEncodingException;
+
+/**
+ * Re-implementation of {@link DeltaBinaryPackingValuesWriterForInteger}
+ */
+public class ParquetDeltaBinaryPackingValuesWriterForInteger extends AbstractParquetDeltaBinaryPackingValuesWriter {
+ /**
+ * max bitwidth for a mini block, it is used to allocate miniBlockByteBuffer which is
+ * reused between flushes.
+ */
+ private static final int MAX_BITWIDTH = 32;
+
+ private final int blockSizeInValues;
+ private final int miniBlockNumInABlock;
+ private final int miniBlockSizeInValues;
+
+ /**
+ * stores delta values starting from the 2nd value written(1st value is stored in header).
+ * It's reused between flushes
+ */
+ private final int[] deltaBlockBuffer;
+
+ /**
+ * firstValue is written to the header of the page
+ */
+ private int firstValue = 0;
+
+ /**
+ * cache previous written value for calculating delta
+ */
+ private int previousValue = 0;
+
+ /**
+ * min delta is written to the beginning of each block.
+ * it's zig-zag encoded. The deltas stored in each block is actually the difference to min delta,
+ * therefore are all positive
+ * it will be reset after each flush
+ */
+ private int minDeltaInCurrentBlock = Integer.MAX_VALUE;
+ private int maxDeltaInCurrentBlock = Integer.MIN_VALUE;
+ private int estimatedSize = 0;
+
+ public ParquetDeltaBinaryPackingValuesWriterForInteger(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ this(DEFAULT_NUM_BLOCK_VALUES, DEFAULT_NUM_MINIBLOCKS, multiPageOpRef);
+ }
+
+ public ParquetDeltaBinaryPackingValuesWriterForInteger(int blockSizeInValues, int miniBlockNum,
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ super(blockSizeInValues, miniBlockNum, multiPageOpRef);
+ this.blockSizeInValues = blockSizeInValues;
+ this.miniBlockNumInABlock = miniBlockNum;
+ double miniSize = (double) blockSizeInValues / miniBlockNumInABlock;
+ Preconditions.checkArgument(miniSize % 8 == 0, "miniBlockSize must be multiple of 8, but it's " + miniSize);
+ this.miniBlockSizeInValues = (int) miniSize;
+
+ deltaBlockBuffer = new int[blockSizeInValues];
+ miniBlockByteBuffer = new byte[miniBlockSizeInValues * MAX_BITWIDTH];
+ }
+
+ @Override
+ public void writeInteger(int v) {
+ totalValueCount++;
+
+ if (totalValueCount == 1) {
+ firstValue = v;
+ previousValue = firstValue;
+ return;
+ }
+
+ // Calculate delta. The possible overflow is accounted for. The algorithm is correct because
+ // Java int is working as a modalar ring with base 2^32 and because of the plus and minus
+ // properties of a ring. http://en.wikipedia.org/wiki/Modular_arithmetic#Integers_modulo_n
+ int delta = v - previousValue;
+ previousValue = v;
+
+ deltaBlockBuffer[deltaValuesToFlush++] = delta;
+
+ if (delta < minDeltaInCurrentBlock) {
+ minDeltaInCurrentBlock = delta;
+ }
+
+ if (blockSizeInValues == deltaValuesToFlush) {
+ flushBlockBuffer();
+ } else {
+ //Recalibrate the estimated size
+ if (delta > maxDeltaInCurrentBlock) {
+ maxDeltaInCurrentBlock = delta;
+ estimatedElementSize =
+ (64 - Long.numberOfLeadingZeros(maxDeltaInCurrentBlock - minDeltaInCurrentBlock));
+ estimatedSize = estimatedElementSize * deltaValuesToFlush;
+ } else {
+ estimatedSize += estimatedElementSize;
+ }
+ }
+ }
+
+ private void flushBlockBuffer() {
+ // since we store the min delta, the deltas will be converted to be the difference to min delta
+ // and all positive
+ for (int i = 0; i < deltaValuesToFlush; i++) {
+ deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaInCurrentBlock;
+ }
+
+ writeMinDelta();
+ int miniBlocksToFlush = getMiniBlockCountToFlush(deltaValuesToFlush);
+
+ calculateBitWidthsForDeltaBlockBuffer(miniBlocksToFlush);
+ for (int i = 0; i < miniBlockNumInABlock; i++) {
+ writeBitWidthForMiniBlock(i);
+ }
+
+ for (int i = 0; i < miniBlocksToFlush; i++) {
+ // writing i th miniblock
+ int currentBitWidth = bitWidths[i];
+ int blockOffset = 0;
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(currentBitWidth);
+ int miniBlockStart = i * miniBlockSizeInValues;
+ for (int j = miniBlockStart; j < (i + 1) * miniBlockSizeInValues; j += 8) {//8 values per pack
+ // mini block is atomic in terms of flushing
+ // This may write more values when reach to the end of data writing to last mini block,
+ // since it may not be aligned to miniblock,
+ // but doesn't matter. The reader uses total count to see if reached the end.
+ packer.pack8Values(deltaBlockBuffer, j, miniBlockByteBuffer, blockOffset);
+ blockOffset += currentBitWidth;
+ }
+ try {
+ outputStream.write(miniBlockByteBuffer, 0, blockOffset);
+ } catch (IOException e) {
+ throw new ParquetEncodingException(e);
+ }
+ }
+
+ minDeltaInCurrentBlock = Integer.MAX_VALUE;
+ deltaValuesToFlush = 0;
+ estimatedSize = 0;
+ maxDeltaInCurrentBlock = Integer.MIN_VALUE;
+ }
+
+ private void writeMinDelta() {
+ try {
+ BytesUtils.writeZigZagVarInt(minDeltaInCurrentBlock, outputStream);
+ } catch (IOException e) {
+ throw new ParquetEncodingException("can not write min delta for block", e);
+ }
+ }
+
+ /**
+ * iterate through values in each mini block and calculate the bitWidths of max values.
+ *
+ * @param miniBlocksToFlush number of miniblocks
+ */
+ private void calculateBitWidthsForDeltaBlockBuffer(int miniBlocksToFlush) {
+ for (int miniBlockIndex = 0; miniBlockIndex < miniBlocksToFlush; miniBlockIndex++) {
+ int mask = 0;
+ int miniStart = miniBlockIndex * miniBlockSizeInValues;
+
+ /*
+ * The end of current mini block could be the end of current block(deltaValuesToFlush) buffer
+ * when data is not aligned to mini block
+ */
+ int miniEnd = Math.min((miniBlockIndex + 1) * miniBlockSizeInValues, deltaValuesToFlush);
+
+ for (int i = miniStart; i < miniEnd; i++) {
+ mask |= deltaBlockBuffer[i];
+ }
+ bitWidths[miniBlockIndex] = 32 - Integer.numberOfLeadingZeros(mask);
+ }
+ }
+
+ /**
+ * getBytes will trigger flushing block buffer, DO NOT write after getBytes() is called without calling reset()
+ *
+ * @return a BytesInput that contains the encoded page data
+ */
+ @Override
+ public BytesInput getBytes() {
+ // The Page Header should include: blockSizeInValues, numberOfMiniBlocks, totalValueCount
+ if (deltaValuesToFlush != 0) {
+ flushBlockBuffer();
+ }
+ BytesInput configBytes = BytesInput.concat(BytesInput.fromUnsignedVarInt(blockSizeInValues),
+ BytesInput.fromUnsignedVarInt(miniBlockNumInABlock));
+ return BytesInput.concat(configBytes, BytesInput.fromUnsignedVarInt(totalValueCount),
+ BytesInput.fromZigZagVarInt(firstValue), outputStream.asBytesInput());
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ super.reset();
+ this.minDeltaInCurrentBlock = Integer.MAX_VALUE;
+ estimatedSize = 0;
+ maxDeltaInCurrentBlock = Integer.MIN_VALUE;
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ this.minDeltaInCurrentBlock = Integer.MAX_VALUE;
+ estimatedSize = 0;
+ maxDeltaInCurrentBlock = Integer.MIN_VALUE;
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaBinaryPackingValuesWriterForLong.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaBinaryPackingValuesWriterForLong.java
new file mode 100644
index 0000000..6ba40c1
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaBinaryPackingValuesWriterForLong.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import java.io.IOException;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong;
+import org.apache.parquet.io.ParquetEncodingException;
+
+/**
+ * Re-implementation of {@link DeltaBinaryPackingValuesWriterForLong}
+ */
+public class ParquetDeltaBinaryPackingValuesWriterForLong extends AbstractParquetDeltaBinaryPackingValuesWriter {
+ /**
+ * max bitwidth for a mini block, it is used to allocate miniBlockByteBuffer which is
+ * reused between flushes.
+ */
+ private static final int MAX_BITWIDTH = 64;
+
+ private final int blockSizeInValues;
+ private final int miniBlockNumInABlock;
+ private final int miniBlockSizeInValues;
+
+ /**
+ * stores delta values starting from the 2nd value written(1st value is stored in header).
+ * It's reused between flushes
+ */
+ private final long[] deltaBlockBuffer;
+
+ /**
+ * firstValue is written to the header of the page
+ */
+ private long firstValue = 0;
+
+ /**
+ * cache previous written value for calculating delta
+ */
+ private long previousValue = 0;
+
+ /**
+ * min delta is written to the beginning of each block.
+ * it's zig-zag encoded. The deltas stored in each block is actually the difference to min delta,
+ * therefore are all positive
+ * it will be reset after each flush
+ */
+ private long minDeltaInCurrentBlock = Long.MAX_VALUE;
+ private long maxDeltaInCurrentBlock = Long.MIN_VALUE;
+
+ public ParquetDeltaBinaryPackingValuesWriterForLong(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ this(DEFAULT_NUM_BLOCK_VALUES, DEFAULT_NUM_MINIBLOCKS, multiPageOpRef);
+ }
+
+ public ParquetDeltaBinaryPackingValuesWriterForLong(int blockSizeInValues, int miniBlockNum,
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ super(blockSizeInValues, miniBlockNum, multiPageOpRef);
+ this.blockSizeInValues = blockSizeInValues;
+ this.miniBlockNumInABlock = miniBlockNum;
+ double miniSize = (double) blockSizeInValues / miniBlockNumInABlock;
+ Preconditions.checkArgument(miniSize % 8 == 0, "miniBlockSize must be multiple of 8, but it's " + miniSize);
+ this.miniBlockSizeInValues = (int) miniSize;
+ deltaBlockBuffer = new long[blockSizeInValues];
+ miniBlockByteBuffer = new byte[miniBlockSizeInValues * MAX_BITWIDTH];
+ }
+
+ @Override
+ public void writeLong(long v) {
+ totalValueCount++;
+
+ if (totalValueCount == 1) {
+ firstValue = v;
+ previousValue = firstValue;
+ return;
+ }
+
+ // Calculate delta. The possible overflow is accounted for. The algorithm is correct because
+ // Java long is working as a modalar ring with base 2^64 and because of the plus and minus
+ // properties of a ring. http://en.wikipedia.org/wiki/Modular_arithmetic#Integers_modulo_n
+ long delta = v - previousValue;
+ previousValue = v;
+
+ deltaBlockBuffer[deltaValuesToFlush++] = delta;
+
+ if (delta < minDeltaInCurrentBlock) {
+ minDeltaInCurrentBlock = delta;
+ }
+
+ if (blockSizeInValues == deltaValuesToFlush) {
+ flushBlockBuffer();
+ } else {
+ //Recalibrate the estimated size
+ if (delta > maxDeltaInCurrentBlock) {
+ maxDeltaInCurrentBlock = delta;
+ estimatedElementSize =
+ (64 - Long.numberOfLeadingZeros(maxDeltaInCurrentBlock - minDeltaInCurrentBlock));
+ estimatedSize = estimatedElementSize * deltaValuesToFlush;
+ } else {
+ estimatedSize += estimatedElementSize;
+ }
+ }
+ }
+
+ private void flushBlockBuffer() {
+ // since we store the min delta, the deltas will be converted to be the difference to min delta
+ // and all positive
+ for (int i = 0; i < deltaValuesToFlush; i++) {
+ deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaInCurrentBlock;
+ }
+
+ writeMinDelta();
+ int miniBlocksToFlush = getMiniBlockCountToFlush(deltaValuesToFlush);
+
+ calculateBitWidthsForDeltaBlockBuffer(miniBlocksToFlush);
+ int minBitWidth = Integer.MAX_VALUE;
+ for (int i = 0; i < miniBlockNumInABlock; i++) {
+ writeBitWidthForMiniBlock(i);
+ minBitWidth = Math.min(bitWidths[i], minBitWidth);
+ }
+
+ for (int i = 0; i < miniBlocksToFlush; i++) {
+ // writing i th miniblock
+ int currentBitWidth = bitWidths[i];
+ int blockOffset = 0;
+ // TODO: should this cache the packer?
+ BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(currentBitWidth);
+ int miniBlockStart = i * miniBlockSizeInValues;
+ // pack values into the miniblock buffer, 8 at a time to get exactly currentBitWidth bytes
+ for (int j = miniBlockStart; j < (i + 1) * miniBlockSizeInValues; j += 8) {
+ // mini block is atomic in terms of flushing
+ // This may write more values when reach to the end of data writing to last mini block,
+ // since it may not be aligned to miniblock,
+ // but doesn't matter. The reader uses total count to see if reached the end.
+ packer.pack8Values(deltaBlockBuffer, j, miniBlockByteBuffer, blockOffset);
+ blockOffset += currentBitWidth;
+ }
+ try {
+ outputStream.write(miniBlockByteBuffer, 0, blockOffset);
+ } catch (IOException e) {
+ throw new ParquetEncodingException(e);
+ }
+ }
+
+ minDeltaInCurrentBlock = Long.MAX_VALUE;
+ maxDeltaInCurrentBlock = Long.MIN_VALUE;
+ deltaValuesToFlush = 0;
+ estimatedElementSize = 0;
+ estimatedSize = 0;
+ }
+
+ private void writeMinDelta() {
+ try {
+ BytesUtils.writeZigZagVarLong(minDeltaInCurrentBlock, outputStream);
+ } catch (IOException e) {
+ throw new ParquetEncodingException("can not write min delta for block", e);
+ }
+ }
+
+ /**
+ * iterate through values in each mini block and calculate the bitWidths of max values.
+ *
+ * @param miniBlocksToFlush number of miniblocks
+ */
+ private void calculateBitWidthsForDeltaBlockBuffer(int miniBlocksToFlush) {
+ for (int miniBlockIndex = 0; miniBlockIndex < miniBlocksToFlush; miniBlockIndex++) {
+ long mask = 0;
+ int miniStart = miniBlockIndex * miniBlockSizeInValues;
+
+ //The end of current mini block could be the end of current block(deltaValuesToFlush) buffer
+ //when data is not aligned to mini block
+ int miniEnd = Math.min((miniBlockIndex + 1) * miniBlockSizeInValues, deltaValuesToFlush);
+
+ for (int i = miniStart; i < miniEnd; i++) {
+ mask |= deltaBlockBuffer[i];
+ }
+ bitWidths[miniBlockIndex] = 64 - Long.numberOfLeadingZeros(mask);
+ }
+ }
+
+ /**
+ * getBytes will trigger flushing block buffer, DO NOT write after getBytes() is called without calling reset()
+ *
+ * @return a BytesInput that contains the encoded page data
+ */
+ @Override
+ public BytesInput getBytes() {
+ // The Page Header should include: blockSizeInValues, numberOfMiniBlocks, totalValueCount
+ if (deltaValuesToFlush != 0) {
+ flushBlockBuffer();
+ }
+ BytesInput configBytes = BytesInput.concat(BytesInput.fromUnsignedVarInt(blockSizeInValues),
+ BytesInput.fromUnsignedVarInt(miniBlockNumInABlock));
+ return BytesInput.concat(configBytes, BytesInput.fromUnsignedVarInt(totalValueCount),
+ BytesInput.fromZigZagVarLong(firstValue), outputStream.asBytesInput());
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ super.reset();
+ this.minDeltaInCurrentBlock = Long.MAX_VALUE;
+ this.maxDeltaInCurrentBlock = Long.MIN_VALUE;
+ previousValue = 0;
+ estimatedElementSize = 0;
+ estimatedSize = 0;
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ this.minDeltaInCurrentBlock = Long.MAX_VALUE;
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaByteArrayWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaByteArrayWriter.java
new file mode 100644
index 0000000..1b46116
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaByteArrayWriter.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter;
+
+/**
+ * Re-implementation of {@link DeltaByteArrayWriter}
+ */
+public class ParquetDeltaByteArrayWriter extends AbstractParquetValuesWriter {
+ private static final IValueReference EMPTY_VALUE;
+ private final ParquetDeltaBinaryPackingValuesWriterForInteger prefixLengthWriter;
+ private final ParquetDeltaLengthByteArrayValuesWriter suffixWriter;
+ private final VoidPointable suffix;
+ private final ArrayBackedValueStorage previous = new ArrayBackedValueStorage();
+
+ static {
+ VoidPointable emptyPointable = new VoidPointable();
+ emptyPointable.set(new byte[0], 0, 0);
+ EMPTY_VALUE = emptyPointable;
+ }
+
+ public ParquetDeltaByteArrayWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ this.prefixLengthWriter = new ParquetDeltaBinaryPackingValuesWriterForInteger(multiPageOpRef);
+ this.suffixWriter = new ParquetDeltaLengthByteArrayValuesWriter(multiPageOpRef);
+ suffix = new VoidPointable();
+ suffix.set(EMPTY_VALUE);
+ }
+
+ @Override
+ public BytesInput getBytes() {
+ BytesInput prefixBytes = prefixLengthWriter.getBytes();
+ BytesInput prefixLength = BytesInput.fromUnsignedVarInt((int) prefixBytes.size());
+ BytesInput suffixBytes = suffixWriter.getBytes();
+ return BytesInput.concat(prefixLength, prefixBytes, suffixBytes);
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ prefixLengthWriter.reset();
+ suffixWriter.reset();
+ previous.reset();
+ suffix.set(EMPTY_VALUE);
+ }
+
+ @Override
+ public void close() {
+ prefixLengthWriter.close();
+ suffixWriter.close();
+ previous.reset();
+ suffix.set(EMPTY_VALUE);
+ }
+
+ @Override
+ public int getEstimatedSize() {
+ return prefixLengthWriter.getEstimatedSize() + suffixWriter.getEstimatedSize();
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return prefixLengthWriter.getAllocatedSize() + suffixWriter.getAllocatedSize();
+ }
+
+ @Override
+ public void writeBytes(IValueReference value, boolean skipLengthBytes) {
+ byte[] bytes = value.getByteArray();
+ int start = value.getStartOffset();
+ int length = value.getLength();
+ if (skipLengthBytes) {
+ int lengthBytes = UTF8StringUtil.getNumBytesToStoreLength(bytes, start);
+ start += lengthBytes;
+ length -= lengthBytes;
+ }
+ writeBytes(bytes, start, length);
+ }
+
+ private void writeBytes(byte[] bytes, int offset, int length) {
+ final byte[] prevBytes = previous.getByteArray();
+ final int prevOffset = previous.getStartOffset();
+ final int minLength = Math.min(length, previous.getLength());
+ // find the number of matching prefix bytes between this value and the previous one
+ int i;
+ for (i = 0; (i < minLength) && (bytes[i + offset] == prevBytes[i + prevOffset]); i++);
+ prefixLengthWriter.writeInteger(i);
+ suffix.set(bytes, offset + i, length - i);
+ suffixWriter.writeBytes(suffix, false);
+ // We store as bytes could be evicted from the buffer cache
+ previous.set(bytes, offset, length);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaLengthByteArrayValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaLengthByteArrayValuesWriter.java
new file mode 100644
index 0000000..afab48eb
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetDeltaLengthByteArrayValuesWriter.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.out.MultiTemporaryBufferBytesOutputStream;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.LittleEndianDataOutputStream;
+import org.apache.parquet.column.values.deltalengthbytearray.DeltaLengthByteArrayValuesWriter;
+import org.apache.parquet.io.ParquetEncodingException;
+
+/**
+ * Re-implementation of {@link DeltaLengthByteArrayValuesWriter}
+ */
+public class ParquetDeltaLengthByteArrayValuesWriter extends AbstractParquetValuesWriter {
+ private final ParquetDeltaBinaryPackingValuesWriterForInteger lengthWriter;
+ private final MultiTemporaryBufferBytesOutputStream outputStream;
+ private final LittleEndianDataOutputStream out;
+
+ public ParquetDeltaLengthByteArrayValuesWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ outputStream = new MultiTemporaryBufferBytesOutputStream(multiPageOpRef);
+ out = new LittleEndianDataOutputStream(outputStream);
+ lengthWriter = new ParquetDeltaBinaryPackingValuesWriterForInteger(multiPageOpRef);
+ }
+
+ @Override
+ public void writeBytes(IValueReference value, boolean skipLengthBytes) {
+ try {
+ lengthWriter.writeInteger(value.getLength());
+ out.write(value.getByteArray(), value.getStartOffset(), value.getLength());
+ } catch (IOException e) {
+ throw new ParquetEncodingException("could not write bytes", e);
+ }
+ }
+
+ @Override
+ public BytesInput getBytes() {
+ try {
+ out.flush();
+ } catch (IOException e) {
+ throw new ParquetEncodingException("could not write page", e);
+ }
+ BytesInput lengthBytes = lengthWriter.getBytes();
+ BytesInput lengthSize = BytesInput.fromUnsignedVarInt((int) lengthBytes.size());
+ BytesInput arrayBytes = outputStream.asBytesInput();
+ return BytesInput.concat(lengthSize, lengthBytes, arrayBytes);
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ lengthWriter.reset();
+ outputStream.reset();
+ }
+
+ @Override
+ public void close() {
+ lengthWriter.close();
+ outputStream.finish();
+ }
+
+ @Override
+ public int getEstimatedSize() {
+ return lengthWriter.getEstimatedSize() + outputStream.size();
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return lengthWriter.getAllocatedSize() + outputStream.capacity();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetPlainValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetPlainValuesWriter.java
new file mode 100644
index 0000000..0298e59
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetPlainValuesWriter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.asterix.column.bytes.stream.out.AbstractBytesOutputStream;
+import org.apache.asterix.column.bytes.stream.out.MultiTemporaryBufferBytesOutputStream;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.LittleEndianDataOutputStream;
+import org.apache.parquet.column.values.plain.PlainValuesWriter;
+import org.apache.parquet.io.ParquetEncodingException;
+
+/**
+ * Re-implementation of {@link PlainValuesWriter}
+ */
+public class ParquetPlainValuesWriter extends AbstractParquetValuesWriter {
+ public static final Charset CHARSET = StandardCharsets.UTF_8;
+
+ private final AbstractBytesOutputStream outputStream;
+ private final LittleEndianDataOutputStream out;
+
+ public ParquetPlainValuesWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ outputStream = new MultiTemporaryBufferBytesOutputStream(multiPageOpRef);
+ out = new LittleEndianDataOutputStream(outputStream);
+ }
+
+ @Override
+ public final void writeDouble(double v) {
+ try {
+ out.writeDouble(v);
+ } catch (IOException e) {
+ throw new ParquetEncodingException("could not write double", e);
+ }
+ }
+
+ @Override
+ public BytesInput getBytes() {
+ try {
+ out.flush();
+ } catch (IOException e) {
+ throw new ParquetEncodingException("could not write page", e);
+ }
+ return outputStream.asBytesInput();
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ outputStream.reset();
+ }
+
+ @Override
+ public void close() {
+ outputStream.finish();
+ }
+
+ @Override
+ public int getEstimatedSize() {
+ return outputStream.size();
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return outputStream.capacity();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetRunLengthBitPackingHybridEncoder.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetRunLengthBitPackingHybridEncoder.java
new file mode 100644
index 0000000..671e0a1
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/encoder/ParquetRunLengthBitPackingHybridEncoder.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.encoder;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.out.GrowableBytesOutputStream;
+import org.apache.asterix.column.bytes.stream.out.pointer.IReservedPointer;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder;
+
+/**
+ * Re-implementation of {@link RunLengthBitPackingHybridEncoder}
+ */
+public class ParquetRunLengthBitPackingHybridEncoder {
+ private final BytePacker packer;
+
+ private final GrowableBytesOutputStream outputStream;
+
+ /**
+ * The bit width used for bit-packing and for writing
+ * the repeated-value
+ */
+ private final int bitWidth;
+
+ /**
+ * Values that are bit packed 8 at at a time are packed into this
+ * buffer, which is then written to baos
+ */
+ private final byte[] packBuffer;
+
+ /**
+ * Previous value written, used to detect repeated values
+ */
+ private int previousValue;
+
+ /**
+ * We buffer 8 values at a time, and either bit pack them
+ * or discard them after writing a rle-run
+ */
+ private final int[] bufferedValues;
+ private int numBufferedValues;
+
+ /**
+ * How many times a value has been repeated
+ */
+ private int repeatCount;
+
+ /**
+ * How many groups of 8 values have been written
+ * to the current bit-packed-run
+ */
+ private int bitPackedGroupCount;
+
+ /**
+ * A "pointer" to a single byte in baos,
+ * which we use as our bit-packed-header. It's really
+ * the logical index of the byte in baos.
+ * <p>
+ * We are only using one byte for this header,
+ * which limits us to writing 504 values per bit-packed-run.
+ * <p>
+ * MSB must be 0 for varint encoding, LSB must be 1 to signify
+ * that this is a bit-packed-header leaves 6 bits to write the
+ * number of 8-groups -> (2^6 - 1) * 8 = 504
+ */
+ private final IReservedPointer bitPackedRunHeaderPointer;
+
+ private boolean toBytesCalled;
+
+ public ParquetRunLengthBitPackingHybridEncoder(int bitWidth) {
+
+ Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");
+
+ this.bitWidth = bitWidth;
+ this.outputStream = new GrowableBytesOutputStream();
+ this.bitPackedRunHeaderPointer = outputStream.createPointer();
+ this.packBuffer = new byte[bitWidth];
+ this.bufferedValues = new int[8];
+ this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ reset(false);
+ }
+
+ private void reset(boolean resetBaos) {
+ if (resetBaos) {
+ this.outputStream.reset();
+ }
+ this.previousValue = 0;
+ this.numBufferedValues = 0;
+ this.repeatCount = 0;
+ this.bitPackedGroupCount = 0;
+ this.bitPackedRunHeaderPointer.reset();
+ this.toBytesCalled = false;
+ }
+
+ public void writeInt(int value) throws IOException {
+ if (value == previousValue) {
+ // keep track of how many times we've seen this value
+ // consecutively
+ ++repeatCount;
+
+ if (repeatCount >= 8) {
+ // we've seen this at least 8 times, we're
+ // certainly going to write an rle-run,
+ // so just keep on counting repeats for now
+ return;
+ }
+ } else {
+ // This is a new value, check if it signals the end of
+ // an rle-run
+ if (repeatCount >= 8) {
+ // it does! write an rle-run
+ writeRleRun();
+ }
+
+ // this is a new value so we've only seen it once
+ repeatCount = 1;
+ // start tracking this value for repeats
+ previousValue = value;
+ }
+
+ // We have not seen enough repeats to justify an rle-run yet,
+ // so buffer this value in case we decide to write a bit-packed-run
+ bufferedValues[numBufferedValues] = value;
+ ++numBufferedValues;
+
+ if (numBufferedValues == 8) {
+ // we've encountered less than 8 repeated values, so
+ // either start a new bit-packed-run or append to the
+ // current bit-packed-run
+ writeOrAppendBitPackedRun();
+ }
+ }
+
+ private void writeOrAppendBitPackedRun() throws IOException {
+ if (bitPackedGroupCount >= 63) {
+ // we've packed as many values as we can for this run,
+ // end it and start a new one
+ endPreviousBitPackedRun();
+ }
+
+ if (!bitPackedRunHeaderPointer.isSet()) {
+ // this is a new bit-packed-run, allocate a byte for the header
+ // and keep a "pointer" to it so that it can be mutated later
+ outputStream.reserveByte(bitPackedRunHeaderPointer);
+ }
+
+ packer.pack8Values(bufferedValues, 0, packBuffer, 0);
+ outputStream.write(packBuffer);
+
+ // empty the buffer, they've all been written
+ numBufferedValues = 0;
+
+ // clear the repeat count, as some repeated values
+ // may have just been bit packed into this run
+ repeatCount = 0;
+
+ ++bitPackedGroupCount;
+ }
+
+ /**
+ * If we are currently writing a bit-packed-run, update the
+ * bit-packed-header and consider this run to be over
+ * <p>
+ * does nothing if we're not currently writing a bit-packed run
+ */
+ private void endPreviousBitPackedRun() {
+ if (!bitPackedRunHeaderPointer.isSet()) {
+ // we're not currently in a bit-packed-run
+ return;
+ }
+
+ // create bit-packed-header, which needs to fit in 1 byte
+ byte bitPackHeader = (byte) ((bitPackedGroupCount << 1) | 1);
+
+ // update this byte
+ bitPackedRunHeaderPointer.setByte(bitPackHeader);
+
+ // mark that this run is over
+ bitPackedRunHeaderPointer.reset();
+
+ // reset the number of groups
+ bitPackedGroupCount = 0;
+ }
+
+ private void writeRleRun() throws IOException {
+ // we may have been working on a bit-packed-run
+ // so close that run if it exists before writing this
+ // rle-run
+ endPreviousBitPackedRun();
+
+ // write the rle-header (lsb of 0 signifies a rle run)
+ BytesUtils.writeUnsignedVarInt(repeatCount << 1, outputStream);
+ // write the repeated-value
+ BytesUtils.writeIntLittleEndianPaddedOnBitWidth(outputStream, previousValue, bitWidth);
+
+ // reset the repeat count
+ repeatCount = 0;
+
+ // throw away all the buffered values, they were just repeats and they've been written
+ numBufferedValues = 0;
+ }
+
+ public BytesInput toBytes() throws IOException {
+ Preconditions.checkArgument(!toBytesCalled, "You cannot call toBytes() more than once without calling reset()");
+
+ // write anything that is buffered / queued up for an rle-run
+ if (repeatCount >= 8) {
+ writeRleRun();
+ } else if (numBufferedValues > 0) {
+ for (int i = numBufferedValues; i < 8; i++) {
+ bufferedValues[i] = 0;
+ }
+ writeOrAppendBitPackedRun();
+ endPreviousBitPackedRun();
+ } else {
+ endPreviousBitPackedRun();
+ }
+
+ toBytesCalled = true;
+ return outputStream.asBytesInput();
+ }
+
+ /**
+ * Reset this encoder for re-use
+ */
+ public void reset() {
+ reset(true);
+ }
+
+ public void close() {
+ reset(false);
+ outputStream.finish();
+ }
+
+ public int getEstimatedSize() {
+ return outputStream.size() + repeatCount * bitWidth;
+ }
+
+ public int getAllocatedSize() {
+ return outputStream.capacity();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/AbstractBytesInputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/AbstractBytesInputStream.java
new file mode 100644
index 0000000..b50143b
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/AbstractBytesInputStream.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.in;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+
+public abstract class AbstractBytesInputStream extends InputStream {
+
+ public abstract void resetAt(int bytesToSkip, AbstractBytesInputStream stream) throws IOException;
+
+ protected abstract void addBuffer(ByteBuffer buffer);
+
+ public abstract void read(IPointable pointable, int length) throws EOFException;
+
+ @Override
+ public abstract int read() throws IOException;
+
+ @Override
+ public abstract int read(byte[] bytes, int offset, int length) throws IOException;
+
+ @Override
+ public abstract long skip(long n);
+
+ public abstract int read(ByteBuffer out);
+
+ public abstract AbstractBytesInputStream remainingStream() throws EOFException;
+
+ public abstract AbstractBytesInputStream sliceStream(int length) throws EOFException;
+
+ @Override
+ public abstract void mark(int readLimit);
+
+ @Override
+ public abstract void reset() throws IOException;
+
+ public abstract void reset(IColumnBufferProvider bufferProvider) throws HyracksDataException;
+
+ @Override
+ public abstract int available();
+
+ public final void skipFully(long n) throws IOException {
+ long skipped = skip(n);
+ if (skipped < n) {
+ throw new EOFException("Not enough bytes to skip: " + skipped + " < " + n);
+ }
+ }
+
+ @Override
+ public final boolean markSupported() {
+ return true;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/ByteBufferInputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/ByteBufferInputStream.java
new file mode 100644
index 0000000..833765c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/ByteBufferInputStream.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.in;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+
+public final class ByteBufferInputStream extends AbstractBytesInputStream {
+ private ByteBuffer buffer;
+ private int mark = -1;
+
+ @Override
+ public void reset(IColumnBufferProvider bufferProvider) {
+ addBuffer(bufferProvider.getBuffer());
+ }
+
+ @Override
+ protected void addBuffer(ByteBuffer buffer) {
+ this.buffer = buffer;
+ mark = -1;
+ }
+
+ @Override
+ public void resetAt(int bytesToSkip, AbstractBytesInputStream stream) throws IOException {
+ ByteBufferInputStream in = (ByteBufferInputStream) stream;
+ buffer = in.buffer.duplicate();
+ buffer.position(buffer.position() + bytesToSkip);
+ mark = -1;
+ }
+
+ @Override
+ public void read(IPointable pointable, int length) throws EOFException {
+ if (buffer.remaining() < length) {
+ throw new EOFException();
+ }
+
+ pointable.set(buffer.array(), buffer.position(), length);
+ buffer.position(buffer.position() + length);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (!buffer.hasRemaining()) {
+ throw new EOFException();
+ }
+ return buffer.get() & 0xFF; // as unsigned
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws IOException {
+ if (length == 0) {
+ return 0;
+ }
+
+ int remaining = buffer.remaining();
+ if (remaining <= 0) {
+ return -1;
+ }
+
+ int bytesToRead = Math.min(remaining, length);
+ buffer.get(bytes, offset, bytesToRead);
+
+ return bytesToRead;
+ }
+
+ @Override
+ public long skip(long n) {
+ if (n == 0) {
+ return 0;
+ }
+
+ if (!buffer.hasRemaining()) {
+ return -1;
+ }
+
+ // buffer.remaining is an int, so this will always fit in an int
+ int bytesToSkip = (int) Math.min(buffer.remaining(), n);
+ buffer.position(buffer.position() + bytesToSkip);
+
+ return bytesToSkip;
+ }
+
+ @Override
+ public int read(ByteBuffer out) {
+ int bytesToCopy;
+ ByteBuffer copyBuffer;
+ if (buffer.remaining() <= out.remaining()) {
+ // copy the entire buffer
+ bytesToCopy = buffer.remaining();
+ copyBuffer = buffer;
+ } else {
+ // copy a slice of the current buffer
+ bytesToCopy = out.remaining();
+ copyBuffer = buffer.duplicate();
+ copyBuffer.position(buffer.position());
+ copyBuffer.limit(buffer.position() + bytesToCopy);
+ buffer.position(buffer.position() + bytesToCopy);
+ }
+
+ out.put(copyBuffer);
+ out.flip();
+
+ return bytesToCopy;
+ }
+
+ @Override
+ public AbstractBytesInputStream sliceStream(int length) throws EOFException {
+ if (buffer.remaining() < length) {
+ throw new EOFException();
+ }
+ ByteBuffer copy = buffer.duplicate();
+ copy.position(buffer.position());
+ copy.limit(buffer.position() + length);
+ ByteBufferInputStream in = new ByteBufferInputStream();
+ in.addBuffer(copy);
+ buffer.position(buffer.position() + length);
+ return in;
+ }
+
+ @Override
+ public AbstractBytesInputStream remainingStream() {
+ ByteBuffer remaining = buffer.duplicate();
+ remaining.position(buffer.position());
+ buffer.position(buffer.limit());
+ ByteBufferInputStream in = new ByteBufferInputStream();
+ in.addBuffer(remaining);
+ return in;
+ }
+
+ @Override
+ public void mark(int readlimit) {
+ this.mark = buffer.position();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ if (mark >= 0) {
+ buffer.position(mark);
+ this.mark = -1;
+ } else {
+ throw new IOException("No mark defined");
+ }
+ }
+
+ @Override
+ public int available() {
+ return buffer.remaining();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java
new file mode 100644
index 0000000..31f8179
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.in;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+
+public final class MultiByteBufferInputStream extends AbstractBytesInputStream {
+ private static final ByteBuffer EMPTY;
+
+ static {
+ EMPTY = ByteBuffer.allocate(0);
+ EMPTY.limit(0);
+ }
+
+ private final Queue<ByteBuffer> buffers;
+ private final ArrayBackedValueStorage tempPointableStorage;
+ private int length;
+
+ private ByteBuffer current;
+ private int position;
+
+ public MultiByteBufferInputStream() {
+ this.buffers = new ArrayDeque<>();
+ tempPointableStorage = new ArrayBackedValueStorage();
+ this.current = EMPTY;
+ this.position = 0;
+ this.length = 0;
+
+ }
+
+ private MultiByteBufferInputStream(MultiByteBufferInputStream original, int len) throws EOFException {
+ buffers = new ArrayDeque<>();
+ tempPointableStorage = new ArrayBackedValueStorage();
+ position = original.position;
+ length = original.length;
+ buffers.addAll(original.sliceBuffers(len));
+ nextBuffer();
+ }
+
+ @Override
+ public void reset(IColumnBufferProvider bufferProvider) throws HyracksDataException {
+ reset();
+ length = bufferProvider.getLength();
+ if (length > 0) {
+ bufferProvider.readAll(buffers);
+ nextBuffer();
+ }
+ }
+
+ @Override
+ protected void addBuffer(ByteBuffer buffer) {
+ buffers.add(buffer);
+ length += buffer.remaining();
+ }
+
+ @Override
+ public void resetAt(int bytesToSkip, AbstractBytesInputStream stream) throws IOException {
+ MultiByteBufferInputStream original = (MultiByteBufferInputStream) stream;
+ buffers.clear();
+ position = original.position;
+ length = original.length;
+ current = original.current.duplicate();
+ for (ByteBuffer buffer : original.buffers) {
+ buffers.add(buffer.duplicate());
+ }
+
+ if (skip(bytesToSkip) != bytesToSkip) {
+ throw new EOFException();
+ }
+ }
+
+ @Override
+ public long skip(long n) {
+ if (n <= 0) {
+ return 0;
+ }
+
+ if (current == null) {
+ return -1;
+ }
+
+ long bytesSkipped = 0;
+ while (bytesSkipped < n) {
+ if (current.remaining() > 0) {
+ long bytesToSkip = Math.min(n - bytesSkipped, current.remaining());
+ current.position(current.position() + (int) bytesToSkip);
+ bytesSkipped += bytesToSkip;
+ this.position += bytesToSkip;
+ } else if (!nextBuffer()) {
+ // there are no more buffers
+ return bytesSkipped > 0 ? bytesSkipped : -1;
+ }
+ }
+
+ return bytesSkipped;
+ }
+
+ @Override
+ public int read(ByteBuffer out) {
+ int len = out.remaining();
+ if (len <= 0) {
+ return 0;
+ }
+
+ if (current == null) {
+ return -1;
+ }
+
+ int bytesCopied = 0;
+ while (bytesCopied < len) {
+ if (current.remaining() > 0) {
+ int bytesToCopy;
+ ByteBuffer copyBuffer;
+ if (current.remaining() <= out.remaining()) {
+ // copy all the current buffer
+ bytesToCopy = current.remaining();
+ copyBuffer = current;
+ } else {
+ // copy a slice of the current buffer
+ bytesToCopy = out.remaining();
+ copyBuffer = current.duplicate();
+ copyBuffer.limit(copyBuffer.position() + bytesToCopy);
+ current.position(copyBuffer.position() + bytesToCopy);
+ }
+
+ out.put(copyBuffer);
+ bytesCopied += bytesToCopy;
+ this.position += bytesToCopy;
+
+ } else if (!nextBuffer()) {
+ // there are no more buffers
+ return bytesCopied > 0 ? bytesCopied : -1;
+ }
+ }
+
+ return bytesCopied;
+ }
+
+ @Override
+ public AbstractBytesInputStream sliceStream(int length) throws EOFException {
+ return new MultiByteBufferInputStream(this, length);
+ }
+
+ @Override
+ public AbstractBytesInputStream remainingStream() throws EOFException {
+ return new MultiByteBufferInputStream(this, length - position);
+ }
+
+ @Override
+ public int read(byte[] bytes, int off, int len) {
+ if (len <= 0) {
+ if (len < 0) {
+ throw new IndexOutOfBoundsException("Read length must be greater than 0: " + len);
+ }
+ return 0;
+ }
+
+ if (current == null) {
+ return -1;
+ }
+
+ int bytesRead = 0;
+ while (bytesRead < len) {
+ if (current.remaining() > 0) {
+ int bytesToRead = Math.min(len - bytesRead, current.remaining());
+ current.get(bytes, off + bytesRead, bytesToRead);
+ bytesRead += bytesToRead;
+ this.position += bytesToRead;
+ } else if (!nextBuffer()) {
+ // there are no more buffers
+ return bytesRead > 0 ? bytesRead : -1;
+ }
+ }
+
+ return bytesRead;
+ }
+
+ @Override
+ public int read(byte[] bytes) {
+ return read(bytes, 0, bytes.length);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (current == null) {
+ throw new EOFException();
+ }
+
+ while (true) {
+ if (current.remaining() > 0) {
+ this.position += 1;
+ return current.get() & 0xFF; // as unsigned
+ } else if (!nextBuffer()) {
+ // there are no more buffers
+ throw new EOFException();
+ }
+ }
+ }
+
+ @Override
+ public void read(IPointable pointable, int length) throws EOFException {
+ if (current.remaining() >= length) {
+ pointable.set(current.array(), current.position(), length);
+ current.position(current.position() + length);
+ position += length;
+ } else {
+ tempPointableStorage.setSize(length);
+ //Read first half part from the current buffer
+ int bytesRead = read(tempPointableStorage.getByteArray(), 0, length);
+ if (bytesRead != length) {
+ throw new EOFException();
+ }
+ pointable.set(tempPointableStorage);
+ }
+ }
+
+ @Override
+ public int available() {
+ return length - position;
+ }
+
+ @Override
+ public void mark(int readLimit) {
+ throw new UnsupportedOperationException("reset() is not supported");
+ }
+
+ @Override
+ public void reset() {
+ buffers.clear();
+ this.current = EMPTY;
+ this.position = 0;
+ this.length = 0;
+ }
+
+ private List<ByteBuffer> sliceBuffers(long length) throws EOFException {
+ if (length <= 0) {
+ return Collections.emptyList();
+ }
+
+ if (current == null) {
+ throw new EOFException();
+ }
+
+ List<ByteBuffer> sliceBuffers = new ArrayList<>();
+ long bytesAccumulated = 0;
+ while (bytesAccumulated < length) {
+ if (current.remaining() > 0) {
+ // get a slice of the current buffer to return
+ // always fits in an int because remaining returns an int that is >= 0
+ int bufLen = (int) Math.min(length - bytesAccumulated, current.remaining());
+ ByteBuffer slice = current.duplicate();
+ slice.limit(slice.position() + bufLen);
+ sliceBuffers.add(slice);
+ bytesAccumulated += bufLen;
+
+ // update state; the bytes are considered read
+ current.position(current.position() + bufLen);
+ this.position += bufLen;
+ } else if (!nextBuffer()) {
+ // there are no more buffers
+ throw new EOFException();
+ }
+ }
+
+ return sliceBuffers;
+ }
+
+ private boolean nextBuffer() {
+ if (buffers.isEmpty()) {
+ return false;
+ }
+ current = buffers.poll();
+ return true;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractBytesOutputStream.java
new file mode 100644
index 0000000..698eac4
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractBytesOutputStream.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.asterix.column.bytes.stream.out.pointer.IReservedPointer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.column.values.ValuesWriter;
+
+/**
+ * Extends {@link OutputStream} to include methods needed by {@link ValuesWriter}
+ */
+public abstract class AbstractBytesOutputStream extends OutputStream {
+ private final ParquetBytesInput bytesInput;
+
+ protected AbstractBytesOutputStream() {
+ bytesInput = new ParquetBytesInput(this);
+ }
+
+ @Override
+ public abstract void write(int b) throws IOException;
+
+ @Override
+ public final void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public abstract void write(byte[] b, int off, int len) throws IOException;
+
+ public final void write(IValueReference value) throws IOException {
+ write(value.getByteArray(), value.getStartOffset(), value.getLength());
+ }
+
+ public final BytesInput asBytesInput() {
+ return bytesInput;
+ }
+
+ public abstract void finish();
+
+ /**
+ * Reset output stream
+ */
+ public abstract void reset() throws HyracksDataException;
+
+ /**
+ * Reserve a byte at the current position of the stream
+ *
+ * @param pointer pointer that references the current position
+ */
+ public abstract void reserveByte(IReservedPointer pointer) throws IOException;
+
+ /**
+ * Reserve an integer at the current position of the stream
+ *
+ * @param pointer pointer that references the current position
+ */
+ public abstract void reserveInteger(IReservedPointer pointer) throws IOException;
+
+ /**
+ * @return a reusable instance of {@link IReservedPointer}
+ */
+ public abstract IReservedPointer createPointer();
+
+ /**
+ * @return Size of written value
+ */
+ public abstract int size();
+
+ /**
+ * @return Allocated buffer size
+ */
+ public abstract int capacity();
+
+ /**
+ * Write the content to another output stream
+ *
+ * @param outputStream output stream to write to
+ */
+ public abstract void writeTo(OutputStream outputStream) throws IOException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractMultiBufferBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractMultiBufferBytesOutputStream.java
new file mode 100644
index 0000000..4b7c835
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractMultiBufferBytesOutputStream.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.column.bytes.stream.out.pointer.ByteBufferReservedPointer;
+import org.apache.asterix.column.bytes.stream.out.pointer.IReservedPointer;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+abstract class AbstractMultiBufferBytesOutputStream extends AbstractBytesOutputStream {
+ protected final Mutable<IColumnWriteMultiPageOp> multiPageOpRef;
+ protected final List<ByteBuffer> buffers;
+ protected int currentBufferIndex;
+ protected int allocatedBytes;
+ protected int position;
+ protected ByteBuffer currentBuf;
+
+ AbstractMultiBufferBytesOutputStream(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ this.multiPageOpRef = multiPageOpRef;
+ buffers = new ArrayList<>();
+ }
+
+ protected abstract ByteBuffer confiscateNewBuffer() throws HyracksDataException;
+
+ protected abstract void preReset() throws HyracksDataException;
+
+ @Override
+ public final void reset() throws HyracksDataException {
+ preReset();
+ position = 0;
+ currentBufferIndex = 0;
+ if (allocatedBytes == 0) {
+ allocateBuffer();
+ }
+ currentBufferIndex = 0;
+ currentBuf = buffers.get(0);
+ currentBuf.clear();
+ }
+
+ @Override
+ public final void write(int b) throws IOException {
+ ensureCapacity(1);
+ currentBuf.put((byte) b);
+ position++;
+ }
+
+ @Override
+ public final void write(byte[] b, int off, int len) throws IOException {
+ ensureCapacity(len);
+ int remaining = len;
+ int offset = off;
+ while (remaining > 0) {
+ setNextBufferIfNeeded();
+ int writeLength = Math.min(remaining, currentBuf.remaining());
+ currentBuf.put(b, offset, writeLength);
+ position += writeLength;
+ offset += writeLength;
+ remaining -= writeLength;
+ }
+ }
+
+ @Override
+ public void reserveByte(IReservedPointer pointer) throws IOException {
+ ensureCapacity(Byte.BYTES);
+ int offset = getCurrentBufferPosition();
+ currentBuf.put((byte) 0);
+ position += 1;
+ ((ByteBufferReservedPointer) pointer).setPointer(currentBuf, offset);
+ }
+
+ @Override
+ public final void reserveInteger(IReservedPointer pointer) throws HyracksDataException {
+ ensureCapacity(Integer.BYTES);
+ int offset = getCurrentBufferPosition();
+ currentBuf.putInt(0);
+ position += Integer.BYTES;
+ ((ByteBufferReservedPointer) pointer).setPointer(currentBuf, offset);
+ }
+
+ @Override
+ public final IReservedPointer createPointer() {
+ return new ByteBufferReservedPointer();
+ }
+
+ public final int getCurrentBufferPosition() {
+ return currentBuf.position();
+ }
+
+ @Override
+ public final int size() {
+ return position;
+ }
+
+ @Override
+ public final int capacity() {
+ return allocatedBytes;
+ }
+
+ @Override
+ public final void finish() {
+ currentBuf = null;
+ buffers.clear();
+ allocatedBytes = 0;
+ }
+
+ /* *************************************************
+ * Helper methods
+ * *************************************************
+ */
+
+ private void ensureCapacity(int length) throws HyracksDataException {
+ if (position + length > allocatedBytes) {
+ allocateMoreBuffers(length);
+ } else if (length > 0) {
+ setNextBufferIfNeeded();
+ }
+ }
+
+ private void allocateMoreBuffers(int length) throws HyracksDataException {
+ int neededSpace = length - currentBuf.remaining();
+ while (neededSpace > 0) {
+ neededSpace -= allocateBuffer();
+ }
+ setNextBufferIfNeeded();
+ }
+
+ private void setNextBufferIfNeeded() {
+ if (currentBuf.remaining() == 0) {
+ currentBuf = buffers.get(++currentBufferIndex);
+ currentBuf.clear();
+ }
+ }
+
+ private int allocateBuffer() throws HyracksDataException {
+ ByteBuffer buffer = confiscateNewBuffer();
+ buffers.add(buffer);
+ buffer.clear();
+ int size = buffer.capacity();
+ allocatedBytes += size;
+ return size;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ByteBufferOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ByteBufferOutputStream.java
new file mode 100644
index 0000000..8817ae6
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ByteBufferOutputStream.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public final class ByteBufferOutputStream extends OutputStream {
+ private ByteBuffer buffer;
+ private int startOffset;
+
+ public void reset(ByteBuffer buffer) {
+ this.buffer = buffer;
+ startOffset = buffer.position();
+ }
+
+ public int size() {
+ return buffer.position() - startOffset;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ buffer.put((byte) b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ buffer.put(b, off, len);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/GrowableBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/GrowableBytesOutputStream.java
new file mode 100644
index 0000000..20daf7d
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/GrowableBytesOutputStream.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.asterix.column.bytes.stream.out.pointer.GrowableBytesPointer;
+import org.apache.asterix.column.bytes.stream.out.pointer.IReservedPointer;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+public final class GrowableBytesOutputStream extends AbstractBytesOutputStream {
+ private final ArrayBackedValueStorage storage;
+
+ public GrowableBytesOutputStream() {
+ storage = new ArrayBackedValueStorage(128);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ storage.getDataOutput().write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ storage.getDataOutput().write(b, off, len);
+ }
+
+ @Override
+ public void finish() {
+ reset();
+ }
+
+ @Override
+ public void reset() {
+ storage.reset();
+ }
+
+ @Override
+ public void reserveByte(IReservedPointer pointer) throws IOException {
+ ((GrowableBytesPointer) pointer).setPointer(storage.getLength());
+ storage.getDataOutput().write(0);
+ }
+
+ @Override
+ public void reserveInteger(IReservedPointer pointer) throws IOException {
+ ((GrowableBytesPointer) pointer).setPointer(storage.getLength());
+ storage.getDataOutput().writeInt(0);
+ }
+
+ @Override
+ public IReservedPointer createPointer() {
+ return new GrowableBytesPointer(storage);
+ }
+
+ @Override
+ public int size() {
+ return storage.getLength();
+ }
+
+ @Override
+ public int capacity() {
+ return storage.getByteArray().length;
+ }
+
+ @Override
+ public void writeTo(OutputStream outputStream) throws IOException {
+ outputStream.write(storage.getByteArray(), storage.getStartOffset(), storage.getLength());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentBufferBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentBufferBytesOutputStream.java
new file mode 100644
index 0000000..c910131
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentBufferBytesOutputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+public final class MultiPersistentBufferBytesOutputStream extends AbstractMultiBufferBytesOutputStream {
+ public MultiPersistentBufferBytesOutputStream(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ super(multiPageOpRef);
+ }
+
+ @Override
+ protected ByteBuffer confiscateNewBuffer() throws HyracksDataException {
+ return multiPageOpRef.getValue().confiscatePersistent();
+ }
+
+ @Override
+ protected void preReset() throws HyracksDataException {
+ if (allocatedBytes > 0) {
+ //Persist all buffers before resetting the stream
+ multiPageOpRef.getValue().persist();
+ allocatedBytes = 0;
+ buffers.clear();
+ }
+ }
+
+ @Override
+ public void writeTo(OutputStream outputStream) {
+ throw new IllegalAccessError("Persistent stream cannot be written to other stream");
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiTemporaryBufferBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiTemporaryBufferBytesOutputStream.java
new file mode 100644
index 0000000..cf2808e
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiTemporaryBufferBytesOutputStream.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+public final class MultiTemporaryBufferBytesOutputStream extends AbstractMultiBufferBytesOutputStream {
+ public MultiTemporaryBufferBytesOutputStream(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ super(multiPageOpRef);
+ }
+
+ @Override
+ protected void preReset() {
+ //NoOp
+ }
+
+ @Override
+ protected ByteBuffer confiscateNewBuffer() throws HyracksDataException {
+ return multiPageOpRef.getValue().confiscateTemporary();
+ }
+
+ @Override
+ public void writeTo(OutputStream outputStream) throws IOException {
+ int writtenSize = 0;
+ for (int i = 0; i < currentBufferIndex + 1; i++) {
+ ByteBuffer buffer = buffers.get(i);
+ outputStream.write(buffer.array(), 0, buffer.position());
+ writtenSize += buffer.position();
+ }
+ if (writtenSize != position) {
+ //Sanity check
+ throw new IllegalStateException("Size is different");
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ParquetBytesInput.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ParquetBytesInput.java
new file mode 100644
index 0000000..c5ad38e
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ParquetBytesInput.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.asterix.column.bytes.encoder.ParquetDeltaBinaryPackingValuesWriterForLong;
+import org.apache.parquet.bytes.BytesInput;
+
+/**
+ * A wrapper for {@link BytesInput} which is used to concatenate multiple {@link AbstractBytesOutputStream}
+ *
+ * @see ParquetDeltaBinaryPackingValuesWriterForLong#getBytes() as an example
+ */
+class ParquetBytesInput extends BytesInput {
+ private final AbstractBytesOutputStream outputStream;
+
+ ParquetBytesInput(AbstractBytesOutputStream outputStream) {
+ this.outputStream = outputStream;
+ }
+
+ @Override
+ public final void writeAllTo(OutputStream outputStream) throws IOException {
+ this.outputStream.writeTo(outputStream);
+ }
+
+ @Override
+ public final long size() {
+ return outputStream.size();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/ByteBufferReservedPointer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/ByteBufferReservedPointer.java
new file mode 100644
index 0000000..8773a31
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/ByteBufferReservedPointer.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out.pointer;
+
+import java.nio.ByteBuffer;
+
+public class ByteBufferReservedPointer implements IReservedPointer {
+ private ByteBuffer buffer;
+ private int offset;
+
+ public void setPointer(ByteBuffer buffer, int offset) {
+ this.buffer = buffer;
+ this.offset = offset;
+ }
+
+ @Override
+ public void setByte(byte value) {
+ buffer.put(offset, value);
+ }
+
+ @Override
+ public void setInteger(int value) {
+ buffer.putInt(offset, value);
+ }
+
+ @Override
+ public void reset() {
+ buffer = null;
+ }
+
+ @Override
+ public boolean isSet() {
+ return buffer != null;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/GrowableBytesPointer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/GrowableBytesPointer.java
new file mode 100644
index 0000000..0863c72
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/GrowableBytesPointer.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out.pointer;
+
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+public class GrowableBytesPointer implements IReservedPointer {
+ private final ArrayBackedValueStorage storage;
+ private int offset;
+
+ public GrowableBytesPointer(ArrayBackedValueStorage storage) {
+ this.storage = storage;
+ }
+
+ public void setPointer(int offset) {
+ this.offset = offset;
+ }
+
+ @Override
+ public void setByte(byte value) {
+ storage.getByteArray()[offset] = value;
+ }
+
+ @Override
+ public void setInteger(int value) {
+ IntegerPointable.setInteger(storage.getByteArray(), offset, value);
+ }
+
+ @Override
+ public void reset() {
+ offset = -1;
+ }
+
+ @Override
+ public boolean isSet() {
+ return offset >= 0;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/IReservedPointer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/IReservedPointer.java
new file mode 100644
index 0000000..46c4d36
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/pointer/IReservedPointer.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out.pointer;
+
+import org.apache.asterix.column.bytes.stream.out.AbstractBytesOutputStream;
+
+/**
+ * Pointer that reference a position in {@link AbstractBytesOutputStream}
+ */
+public interface IReservedPointer {
+ /**
+ * Set a byte value at the pointer's position
+ *
+ * @param value byte value to be set
+ */
+ void setByte(byte value);
+
+ /**
+ * Set an integer value at the pointer's position
+ *
+ * @param value integer value to be set
+ */
+ void setInteger(int value);
+
+ /**
+ * Reset the pointer
+ */
+ void reset();
+
+ /**
+ * @return whether the pointer is set or not
+ */
+ boolean isSet();
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableMetadata.java
new file mode 100644
index 0000000..c7b4651
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableMetadata.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public abstract class AbstractColumnImmutableMetadata extends AbstractColumnMetadata {
+ protected final IValueReference serializedMetadata;
+ protected final int numberOfColumns;
+
+ protected AbstractColumnImmutableMetadata(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ IValueReference serializedMetadata, int numberOfColumns) {
+ super(datasetType, metaType, numberOfPrimaryKeys);
+ this.serializedMetadata = serializedMetadata;
+ this.numberOfColumns = numberOfColumns;
+ }
+
+ @Override
+ public final IValueReference serializeColumnsMetadata() {
+ return serializedMetadata;
+ }
+
+ @Override
+ public final void abort() throws HyracksDataException {
+ //NoOp as the metadata is immutable
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return numberOfColumns;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableReadMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableReadMetadata.java
new file mode 100644
index 0000000..5ac38d7
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableReadMetadata.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+
+public abstract class AbstractColumnImmutableReadMetadata extends AbstractColumnImmutableMetadata
+ implements IColumnProjectionInfo {
+ protected AbstractColumnImmutableReadMetadata(ARecordType datasetType, ARecordType metaType,
+ int numberOfPrimaryKeys, IValueReference serializedMetadata, int numberOfColumns) {
+ super(datasetType, metaType, numberOfPrimaryKeys, serializedMetadata, numberOfColumns);
+ }
+
+ /**
+ * @return the corresponding reader (merge reader or query reader) given <code>this</code> metadata
+ */
+ public abstract AbstractColumnTupleReader createTupleReader();
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnMetadata.java
new file mode 100644
index 0000000..4e19cbc
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnMetadata.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+
+public abstract class AbstractColumnMetadata implements IColumnMetadata {
+ protected static final int WRITERS_POINTER = 0;
+ protected static final int FIELD_NAMES_POINTER = WRITERS_POINTER + Integer.BYTES;
+ protected static final int SCHEMA_POINTER = FIELD_NAMES_POINTER + Integer.BYTES;
+ protected static final int META_SCHEMA_POINTER = SCHEMA_POINTER + Integer.BYTES;
+ protected static final int PATH_INFO_POINTER = META_SCHEMA_POINTER + Integer.BYTES;
+ protected static final int OFFSETS_SIZE = PATH_INFO_POINTER + Integer.BYTES;
+ private final ARecordType datasetType;
+ private final ARecordType metaType;
+
+ private final int numberOfPrimaryKeys;
+ private final int recordFieldIndex;
+
+ protected AbstractColumnMetadata(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys) {
+ this.datasetType = datasetType;
+ this.metaType = metaType;
+ this.numberOfPrimaryKeys = numberOfPrimaryKeys;
+ this.recordFieldIndex = numberOfPrimaryKeys;
+ }
+
+ public final ARecordType getDatasetType() {
+ return datasetType;
+ }
+
+ public final ARecordType getMetaType() {
+ return metaType;
+ }
+
+ public final int getNumberOfPrimaryKeys() {
+ return numberOfPrimaryKeys;
+ }
+
+ public final int getRecordFieldIndex() {
+ return recordFieldIndex;
+ }
+
+ public final int getMetaRecordFieldIndex() {
+ return recordFieldIndex + 1;
+ }
+
+ public abstract int getNumberOfColumns();
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/FieldNamesDictionary.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/FieldNamesDictionary.java
new file mode 100644
index 0000000..aa2e194
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/FieldNamesDictionary.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2IntMap;
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+
+public class FieldNamesDictionary {
+ //For both declared and inferred fields
+ private final List<IValueReference> fieldNames;
+ private final Object2IntMap<String> declaredFieldNamesToIndexMap;
+ private final Int2IntMap hashToFieldNameIndexMap;
+ private final IBinaryHashFunction fieldNameHashFunction;
+
+ //For declared fields
+ private final AMutableString mutableString;
+ private final AStringSerializerDeserializer stringSerDer;
+
+ //For lookups
+ private final ArrayBackedValueStorage lookupStorage;
+
+ public FieldNamesDictionary() {
+ this(new ArrayList<>(), new Object2IntOpenHashMap<>(), new Int2IntOpenHashMap());
+ }
+
+ private FieldNamesDictionary(List<IValueReference> fieldNames, Object2IntMap<String> declaredFieldNamesToIndexMap,
+ Int2IntMap hashToFieldNameIndexMap) {
+ this.fieldNames = fieldNames;
+ this.declaredFieldNamesToIndexMap = declaredFieldNamesToIndexMap;
+ this.hashToFieldNameIndexMap = hashToFieldNameIndexMap;
+
+ mutableString = new AMutableString("");
+ stringSerDer = new AStringSerializerDeserializer(new UTF8StringWriter(), new UTF8StringReader());
+ fieldNameHashFunction =
+ new PointableBinaryHashFunctionFactory(UTF8StringPointable.FACTORY).createBinaryHashFunction();
+ lookupStorage = new ArrayBackedValueStorage();
+ }
+
+ public List<IValueReference> getFieldNames() {
+ return fieldNames;
+ }
+
+ //TODO solve collision (they're so rare that I haven't seen any)
+ public int getOrCreateFieldNameIndex(IValueReference fieldName) throws HyracksDataException {
+ int hash = getHash(fieldName);
+ if (!hashToFieldNameIndexMap.containsKey(hash)) {
+ int index = addFieldName(creatFieldName(fieldName), hash);
+ hashToFieldNameIndexMap.put(hash, index);
+ return index;
+ }
+ return hashToFieldNameIndexMap.get(hash);
+ }
+
+ public int getOrCreateFieldNameIndex(String fieldName) throws HyracksDataException {
+ if (!declaredFieldNamesToIndexMap.containsKey(fieldName)) {
+ IValueReference serializedFieldName = creatFieldName(fieldName);
+ int hash = getHash(serializedFieldName);
+ int index = addFieldName(serializedFieldName, hash);
+ declaredFieldNamesToIndexMap.put(fieldName, index);
+ return index;
+ }
+ return declaredFieldNamesToIndexMap.getInt(fieldName);
+ }
+
+ public int getFieldNameIndex(String fieldName) throws HyracksDataException {
+ lookupStorage.reset();
+ serializeFieldName(fieldName, lookupStorage);
+ return hashToFieldNameIndexMap.getOrDefault(getHash(lookupStorage), -1);
+ }
+
+ private ArrayBackedValueStorage creatFieldName(IValueReference fieldName) throws HyracksDataException {
+ ArrayBackedValueStorage copy = new ArrayBackedValueStorage(fieldName.getLength());
+ copy.append(fieldName);
+ return copy;
+ }
+
+ private ArrayBackedValueStorage creatFieldName(String fieldName) throws HyracksDataException {
+ ArrayBackedValueStorage serializedFieldName = new ArrayBackedValueStorage();
+ serializeFieldName(fieldName, serializedFieldName);
+ return serializedFieldName;
+ }
+
+ private void serializeFieldName(String fieldName, ArrayBackedValueStorage storage) throws HyracksDataException {
+ mutableString.setValue(fieldName);
+ stringSerDer.serialize(mutableString, storage.getDataOutput());
+ }
+
+ private int getHash(IValueReference fieldName) throws HyracksDataException {
+ byte[] object = fieldName.getByteArray();
+ int start = fieldName.getStartOffset();
+ int length = fieldName.getLength();
+
+ return fieldNameHashFunction.hash(object, start, length);
+ }
+
+ private int addFieldName(IValueReference fieldName, int hash) {
+ int index = fieldNames.size();
+ hashToFieldNameIndexMap.put(hash, index);
+ fieldNames.add(fieldName);
+ return index;
+ }
+
+ public IValueReference getFieldName(int index) {
+ return fieldNames.get(index);
+ }
+
+ public void serialize(DataOutput output) throws IOException {
+ output.writeInt(fieldNames.size());
+ for (IValueReference fieldName : fieldNames) {
+ output.writeInt(fieldName.getLength());
+ output.write(fieldName.getByteArray(), fieldName.getStartOffset(), fieldName.getLength());
+ }
+
+ output.writeInt(declaredFieldNamesToIndexMap.size());
+ for (Object2IntMap.Entry<String> declaredFieldIndex : declaredFieldNamesToIndexMap.object2IntEntrySet()) {
+ output.writeUTF(declaredFieldIndex.getKey());
+ output.writeInt(declaredFieldIndex.getIntValue());
+ }
+
+ for (Int2IntMap.Entry hashIndex : hashToFieldNameIndexMap.int2IntEntrySet()) {
+ output.writeInt(hashIndex.getIntKey());
+ output.writeInt(hashIndex.getIntValue());
+ }
+ }
+
+ public static FieldNamesDictionary deserialize(DataInput input) throws IOException {
+ int numberOfFieldNames = input.readInt();
+
+ List<IValueReference> fieldNames = new ArrayList<>();
+ deserializeFieldNames(input, fieldNames, numberOfFieldNames);
+
+ Object2IntMap<String> declaredFieldNamesToIndexMap = new Object2IntOpenHashMap<>();
+ deserializeDeclaredFieldNames(input, declaredFieldNamesToIndexMap);
+
+ Int2IntMap hashToFieldNameIndexMap = new Int2IntOpenHashMap();
+ deserializeHashToFieldNameIndex(input, hashToFieldNameIndexMap, numberOfFieldNames);
+
+ return new FieldNamesDictionary(fieldNames, declaredFieldNamesToIndexMap, hashToFieldNameIndexMap);
+ }
+
+ public void abort(DataInputStream input) throws IOException {
+ int numberOfFieldNames = input.readInt();
+
+ fieldNames.clear();
+ deserializeFieldNames(input, fieldNames, numberOfFieldNames);
+
+ declaredFieldNamesToIndexMap.clear();
+ deserializeDeclaredFieldNames(input, declaredFieldNamesToIndexMap);
+
+ hashToFieldNameIndexMap.clear();
+ deserializeHashToFieldNameIndex(input, hashToFieldNameIndexMap, numberOfFieldNames);
+ }
+
+ private static void deserializeFieldNames(DataInput input, List<IValueReference> fieldNames, int numberOfFieldNames)
+ throws IOException {
+
+ for (int i = 0; i < numberOfFieldNames; i++) {
+ int length = input.readInt();
+ ArrayBackedValueStorage fieldName = new ArrayBackedValueStorage(length);
+ fieldName.setSize(length);
+ input.readFully(fieldName.getByteArray(), 0, length);
+ fieldNames.add(fieldName);
+ }
+ }
+
+ private static void deserializeDeclaredFieldNames(DataInput input,
+ Object2IntMap<String> declaredFieldNamesToIndexMap) throws IOException {
+ int numberOfDeclaredFieldNames = input.readInt();
+ for (int i = 0; i < numberOfDeclaredFieldNames; i++) {
+ String fieldName = input.readUTF();
+ int fieldNameIndex = input.readInt();
+ declaredFieldNamesToIndexMap.put(fieldName, fieldNameIndex);
+ }
+ }
+
+ private static void deserializeHashToFieldNameIndex(DataInput input, Int2IntMap hashToFieldNameIndexMap,
+ int numberOfFieldNames) throws IOException {
+ for (int i = 0; i < numberOfFieldNames; i++) {
+ int hash = input.readInt();
+ int fieldNameIndex = input.readInt();
+ hashToFieldNameIndexMap.put(hash, fieldNameIndex);
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/PathInfoSerializer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/PathInfoSerializer.java
new file mode 100644
index 0000000..f72b77b
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/PathInfoSerializer.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+public class PathInfoSerializer {
+ private final ArrayBackedValueStorage primaryKeyOutputPathStorage;
+ private final ArrayBackedValueStorage pathOutputStorage;
+ private final IntList delimiters;
+ private int level;
+
+ public PathInfoSerializer() {
+ primaryKeyOutputPathStorage = new ArrayBackedValueStorage();
+ pathOutputStorage = new ArrayBackedValueStorage();
+ delimiters = new IntArrayList();
+ level = 0;
+ }
+
+ public void reset() {
+ primaryKeyOutputPathStorage.reset();
+ pathOutputStorage.reset();
+ }
+
+ public void enter(AbstractSchemaNestedNode nestedNode) {
+ if (nestedNode.isCollection()) {
+ delimiters.add(0, level - 1);
+ }
+ if (nestedNode.isObjectOrCollection()) {
+ level++;
+ }
+ }
+
+ public void exit(AbstractSchemaNestedNode nestedNode) {
+ if (nestedNode.isCollection()) {
+ delimiters.removeInt(0);
+ }
+ if (nestedNode.isObjectOrCollection()) {
+ level--;
+ }
+ }
+
+ public void writePathInfo(ATypeTag typeTag, int columnIndex, boolean primaryKey) throws IOException {
+ DataOutput output =
+ primaryKey ? primaryKeyOutputPathStorage.getDataOutput() : pathOutputStorage.getDataOutput();
+ //type tag
+ output.write(typeTag.serialize());
+ //columnIndex
+ output.writeInt(columnIndex);
+ //maxLevel
+ output.writeInt(level);
+ //is primary key
+ output.writeBoolean(primaryKey);
+ //Is collection
+ boolean collection = !delimiters.isEmpty();
+ output.writeBoolean(collection);
+ if (collection) {
+ output.writeInt(delimiters.size());
+ for (int i = 0; i < delimiters.size(); i++) {
+ output.writeInt(delimiters.getInt(i));
+ }
+ }
+ }
+
+ public void serialize(DataOutput output, int numberOfColumns) throws IOException {
+ output.writeInt(numberOfColumns);
+ output.write(primaryKeyOutputPathStorage.getByteArray(), 0, primaryKeyOutputPathStorage.getLength());
+ output.write(pathOutputStorage.getByteArray(), 0, pathOutputStorage.getLength());
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNestedNode.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNestedNode.java
index 3c1a24d..187e460 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNestedNode.java
@@ -16,20 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.metadata.schema;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+public abstract class AbstractSchemaNestedNode extends AbstractSchemaNode {
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+ @Override
+ public final boolean isNested() {
+ return true;
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNode.java
new file mode 100644
index 0000000..c9d8635
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNode.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.PathInfoSerializer;
+import org.apache.asterix.column.metadata.schema.collection.ArraySchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.MultisetSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.MissingFieldSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public abstract class AbstractSchemaNode {
+ private int counter;
+
+ public abstract ATypeTag getTypeTag();
+
+ public abstract boolean isNested();
+
+ public abstract boolean isObjectOrCollection();
+
+ public abstract boolean isCollection();
+
+ public final void incrementCounter() {
+ counter++;
+ }
+
+ public final void setCounter(int counter) {
+ this.counter = counter;
+ }
+
+ public final int getCounter() {
+ return counter;
+ }
+
+ public abstract <R, T> R accept(ISchemaNodeVisitor<R, T> visitor, T arg) throws HyracksDataException;
+
+ public abstract void serialize(DataOutput output, PathInfoSerializer pathInfoSerializer) throws IOException;
+
+ public static AbstractSchemaNode deserialize(DataInput input,
+ Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels) throws IOException {
+ ATypeTag typeTag = ATypeTag.VALUE_TYPE_MAPPING[input.readByte()];
+ switch (typeTag) {
+ case SYSTEM_NULL:
+ return MissingFieldSchemaNode.INSTANCE;
+ case OBJECT:
+ return new ObjectSchemaNode(input, definitionLevels);
+ case ARRAY:
+ return new ArraySchemaNode(input, definitionLevels);
+ case MULTISET:
+ return new MultisetSchemaNode(input, definitionLevels);
+ case UNION:
+ return new UnionSchemaNode(input, definitionLevels);
+ case NULL:
+ case MISSING:
+ case BOOLEAN:
+ case BIGINT:
+ case DOUBLE:
+ case STRING:
+ case UUID:
+ return new PrimitiveSchemaNode(typeTag, input);
+ default:
+ throw new UnsupportedEncodingException(typeTag + " is not supported");
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ISchemaNodeVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ISchemaNodeVisitor.java
new file mode 100644
index 0000000..4d38156
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ISchemaNodeVisitor.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema;
+
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface ISchemaNodeVisitor<R, T> {
+ R visit(ObjectSchemaNode objectNode, T arg) throws HyracksDataException;
+
+ R visit(AbstractCollectionSchemaNode collectionNode, T arg) throws HyracksDataException;
+
+ R visit(UnionSchemaNode unionNode, T arg) throws HyracksDataException;
+
+ R visit(PrimitiveSchemaNode primitiveNode, T arg) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java
new file mode 100644
index 0000000..a230e86
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.PathInfoSerializer;
+import org.apache.asterix.column.metadata.schema.primitive.MissingFieldSchemaNode;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.util.annotations.CriticalPath;
+
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+import it.unimi.dsi.fastutil.ints.Int2IntMap.Entry;
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntImmutableList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+public final class ObjectSchemaNode extends AbstractSchemaNestedNode {
+ private final Int2IntMap fieldNameIndexToChildIndexMap;
+ private final List<AbstractSchemaNode> children;
+
+ public ObjectSchemaNode() {
+ fieldNameIndexToChildIndexMap = new Int2IntOpenHashMap();
+ children = new ArrayList<>();
+ }
+
+ ObjectSchemaNode(DataInput input, Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels)
+ throws IOException {
+ if (definitionLevels != null) {
+ definitionLevels.put(this, new RunLengthIntArray());
+ }
+ int numberOfChildren = input.readInt();
+
+ fieldNameIndexToChildIndexMap = new Int2IntOpenHashMap();
+ deserializeFieldNameIndexToChildIndex(input, fieldNameIndexToChildIndexMap, numberOfChildren);
+
+ children = new ArrayList<>();
+ deserializeChildren(input, children, numberOfChildren, definitionLevels);
+ }
+
+ public AbstractSchemaNode getOrCreateChild(IValueReference fieldName, ATypeTag childTypeTag,
+ FlushColumnMetadata columnMetadata) throws HyracksDataException {
+ int numberOfChildren = children.size();
+ int fieldNameIndex = columnMetadata.getFieldNamesDictionary().getOrCreateFieldNameIndex(fieldName);
+ int childIndex = fieldNameIndexToChildIndexMap.getOrDefault(fieldNameIndex, numberOfChildren);
+ AbstractSchemaNode currentChild = childIndex == numberOfChildren ? null : children.get(childIndex);
+ AbstractSchemaNode newChild = columnMetadata.getOrCreateChild(currentChild, childTypeTag);
+ if (currentChild == null) {
+ children.add(childIndex, newChild);
+ fieldNameIndexToChildIndexMap.put(fieldNameIndex, childIndex);
+ } else if (currentChild != newChild) {
+ children.set(childIndex, newChild);
+ }
+
+ return newChild;
+ }
+
+ public void addChild(int fieldNameIndex, AbstractSchemaNode child) {
+ int childIndex = children.size();
+ fieldNameIndexToChildIndexMap.put(fieldNameIndex, childIndex);
+ children.add(child);
+ }
+
+ public AbstractSchemaNode getChild(int fieldNameIndex) {
+ if (fieldNameIndexToChildIndexMap.containsKey(fieldNameIndex)) {
+ return children.get(fieldNameIndexToChildIndexMap.get(fieldNameIndex));
+ }
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+
+ public void removeChild(int fieldNameIndex) {
+ int childIndex = fieldNameIndexToChildIndexMap.remove(fieldNameIndex);
+ children.remove(childIndex);
+ }
+
+ public List<AbstractSchemaNode> getChildren() {
+ return children;
+ }
+
+ /**
+ * Should not be used in a {@link CriticalPath}
+ */
+ public IntList getChildrenFieldNameIndexes() {
+ return IntImmutableList.toList(fieldNameIndexToChildIndexMap.int2IntEntrySet().stream()
+ .sorted(Comparator.comparingInt(Entry::getIntValue)).mapToInt(Entry::getIntKey));
+ }
+
+ public boolean containsField(int fieldNameIndex) {
+ return fieldNameIndexToChildIndexMap.containsKey(fieldNameIndex);
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.OBJECT;
+ }
+
+ @Override
+ public boolean isObjectOrCollection() {
+ return true;
+ }
+
+ @Override
+ public boolean isCollection() {
+ return false;
+ }
+
+ @Override
+ public <R, T> R accept(ISchemaNodeVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ public void serialize(DataOutput output, PathInfoSerializer pathInfoSerializer) throws IOException {
+ output.write(ATypeTag.OBJECT.serialize());
+ output.writeInt(children.size());
+ for (Int2IntMap.Entry fieldNameIndexChildIndex : fieldNameIndexToChildIndexMap.int2IntEntrySet()) {
+ output.writeInt(fieldNameIndexChildIndex.getIntKey());
+ output.writeInt(fieldNameIndexChildIndex.getIntValue());
+ }
+ pathInfoSerializer.enter(this);
+ for (AbstractSchemaNode child : children) {
+ child.serialize(output, pathInfoSerializer);
+ }
+ pathInfoSerializer.exit(this);
+ }
+
+ public void abort(DataInputStream input, Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels)
+ throws IOException {
+ definitionLevels.put(this, new RunLengthIntArray());
+
+ int numberOfChildren = input.readInt();
+
+ fieldNameIndexToChildIndexMap.clear();
+ deserializeFieldNameIndexToChildIndex(input, fieldNameIndexToChildIndexMap, numberOfChildren);
+
+ children.clear();
+ deserializeChildren(input, children, numberOfChildren, definitionLevels);
+ }
+
+ private static void deserializeFieldNameIndexToChildIndex(DataInput input, Int2IntMap fieldNameIndexToChildIndexMap,
+ int numberOfChildren) throws IOException {
+ for (int i = 0; i < numberOfChildren; i++) {
+ int fieldNameIndex = input.readInt();
+ int childIndex = input.readInt();
+ fieldNameIndexToChildIndexMap.put(fieldNameIndex, childIndex);
+ }
+ }
+
+ private static void deserializeChildren(DataInput input, List<AbstractSchemaNode> children, int numberOfChildren,
+ Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels) throws IOException {
+ for (int i = 0; i < numberOfChildren; i++) {
+ children.add(AbstractSchemaNode.deserialize(input, definitionLevels));
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
new file mode 100644
index 0000000..eba5ac0
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumMap;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.PathInfoSerializer;
+import org.apache.asterix.column.metadata.schema.primitive.MissingFieldSchemaNode;
+import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public final class UnionSchemaNode extends AbstractSchemaNestedNode {
+ private final AbstractSchemaNode originalType;
+ private final Map<ATypeTag, AbstractSchemaNode> children;
+
+ public UnionSchemaNode(AbstractSchemaNode child1, AbstractSchemaNode child2) {
+ children = new EnumMap<>(ATypeTag.class);
+ originalType = child1;
+ putChild(child1);
+ putChild(child2);
+ }
+
+ UnionSchemaNode(DataInput input, Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels)
+ throws IOException {
+ if (definitionLevels != null) {
+ definitionLevels.put(this, new RunLengthIntArray());
+ }
+ ATypeTag originalTypeTag = ATypeTag.VALUE_TYPE_MAPPING[input.readByte()];
+ int numberOfChildren = input.readInt();
+ children = new EnumMap<>(ATypeTag.class);
+ for (int i = 0; i < numberOfChildren; i++) {
+ AbstractSchemaNode child = AbstractSchemaNode.deserialize(input, definitionLevels);
+ children.put(child.getTypeTag(), child);
+ }
+ originalType = children.get(originalTypeTag);
+ }
+
+ private void putChild(AbstractSchemaNode child) {
+ children.put(child.getTypeTag(), child);
+ }
+
+ public AbstractSchemaNode getOriginalType() {
+ return originalType;
+ }
+
+ public AbstractSchemaNode getOrCreateChild(ATypeTag childTypeTag, FlushColumnMetadata columnMetadata)
+ throws HyracksDataException {
+ ATypeTag normalizedTypeTag = FlushColumnMetadata.getNormalizedTypeTag(childTypeTag);
+ AbstractSchemaNode currentChild = children.get(normalizedTypeTag);
+ //The parent of a union child should be the actual parent
+ AbstractSchemaNode newChild = columnMetadata.getOrCreateChild(currentChild, normalizedTypeTag);
+ if (currentChild != newChild) {
+ putChild(newChild);
+ }
+ return newChild;
+ }
+
+ public AbstractSchemaNode getChild(ATypeTag typeTag) {
+ return children.getOrDefault(typeTag, MissingFieldSchemaNode.INSTANCE);
+ }
+
+ public Map<ATypeTag, AbstractSchemaNode> getChildren() {
+ return children;
+ }
+
+ @Override
+ public boolean isObjectOrCollection() {
+ return false;
+ }
+
+ @Override
+ public boolean isCollection() {
+ return false;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.UNION;
+ }
+
+ @Override
+ public <R, T> R accept(ISchemaNodeVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ public void serialize(DataOutput output, PathInfoSerializer pathInfoSerializer) throws IOException {
+ output.write(ATypeTag.UNION.serialize());
+ output.writeByte(originalType.getTypeTag().serialize());
+ output.writeInt(children.size());
+ pathInfoSerializer.enter(this);
+ for (AbstractSchemaNode child : children.values()) {
+ child.serialize(output, pathInfoSerializer);
+ }
+ pathInfoSerializer.exit(this);
+ }
+
+ /**
+ * This would return any numeric node
+ *
+ * @return first numeric node or missing node
+ * @see SchemaClipperVisitor
+ */
+ public AbstractSchemaNode getNumericChildOrMissing() {
+ for (AbstractSchemaNode node : children.values()) {
+ if (ATypeHierarchy.getTypeDomain(node.getTypeTag()) == ATypeHierarchy.Domain.NUMERIC) {
+ return node;
+ }
+ }
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+
+ public int getNumberOfNumericChildren() {
+ int counter = 0;
+ for (AbstractSchemaNode node : children.values()) {
+ if (ATypeHierarchy.getTypeDomain(node.getTypeTag()) == ATypeHierarchy.Domain.NUMERIC) {
+ counter++;
+ }
+ }
+
+ return counter;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/AbstractCollectionSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/AbstractCollectionSchemaNode.java
new file mode 100644
index 0000000..8455864
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/AbstractCollectionSchemaNode.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.collection;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.PathInfoSerializer;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ISchemaNodeVisitor;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public abstract class AbstractCollectionSchemaNode extends AbstractSchemaNestedNode {
+ private AbstractSchemaNode item;
+
+ AbstractCollectionSchemaNode() {
+ item = null;
+ }
+
+ AbstractCollectionSchemaNode(DataInput input, Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels)
+ throws IOException {
+ if (definitionLevels != null) {
+ definitionLevels.put(this, new RunLengthIntArray());
+ }
+ item = AbstractSchemaNode.deserialize(input, definitionLevels);
+ }
+
+ public final AbstractSchemaNode getOrCreateItem(ATypeTag childTypeTag, FlushColumnMetadata columnMetadata)
+ throws HyracksDataException {
+ AbstractSchemaNode newItem = columnMetadata.getOrCreateChild(item, childTypeTag);
+ if (newItem != item) {
+ item = newItem;
+ }
+ return item;
+ }
+
+ public final AbstractSchemaNode getItemNode() {
+ return item;
+ }
+
+ public final void setItemNode(AbstractSchemaNode item) {
+ this.item = item;
+ }
+
+ @Override
+ public final <R, T> R accept(ISchemaNodeVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ public final boolean isObjectOrCollection() {
+ return true;
+ }
+
+ @Override
+ public final boolean isCollection() {
+ return true;
+ }
+
+ @Override
+ public final void serialize(DataOutput output, PathInfoSerializer pathInfoSerializer) throws IOException {
+ output.write(getTypeTag().serialize());
+ pathInfoSerializer.enter(this);
+ item.serialize(output, pathInfoSerializer);
+ pathInfoSerializer.exit(this);
+ }
+
+ public static AbstractCollectionSchemaNode create(ATypeTag typeTag) {
+ if (typeTag == ATypeTag.ARRAY) {
+ return new ArraySchemaNode();
+ }
+
+ return new MultisetSchemaNode();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/ArraySchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/ArraySchemaNode.java
new file mode 100644
index 0000000..084a434
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/ArraySchemaNode.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.collection;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.om.types.ATypeTag;
+
+public final class ArraySchemaNode extends AbstractCollectionSchemaNode {
+
+ public ArraySchemaNode() {
+ super();
+ }
+
+ public ArraySchemaNode(DataInput input, Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels)
+ throws IOException {
+ super(input, definitionLevels);
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.ARRAY;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/MultisetSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/MultisetSchemaNode.java
new file mode 100644
index 0000000..af27a5a
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/MultisetSchemaNode.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.collection;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.om.types.ATypeTag;
+
+public final class MultisetSchemaNode extends AbstractCollectionSchemaNode {
+ public MultisetSchemaNode() {
+ super();
+ }
+
+ public MultisetSchemaNode(DataInput input, Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels)
+ throws IOException {
+ super(input, definitionLevels);
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.MULTISET;
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/MissingFieldSchemaNode.java
similarity index 62%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/MissingFieldSchemaNode.java
index 3c1a24d..98f408e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/MissingFieldSchemaNode.java
@@ -16,20 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.metadata.schema.primitive;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.om.types.ATypeTag;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * A special schema node a non-existing object or union field
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+public final class MissingFieldSchemaNode extends PrimitiveSchemaNode {
+ public static final AbstractSchemaNode INSTANCE = new MissingFieldSchemaNode();
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+ private MissingFieldSchemaNode() {
+ super(-1, ATypeTag.MISSING, false);
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/PrimitiveSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/PrimitiveSchemaNode.java
new file mode 100644
index 0000000..28d379d
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/PrimitiveSchemaNode.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.primitive;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.column.metadata.PathInfoSerializer;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ISchemaNodeVisitor;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class PrimitiveSchemaNode extends AbstractSchemaNode {
+ private final int columnIndex;
+ private final ATypeTag typeTag;
+ private final boolean primaryKey;
+
+ public PrimitiveSchemaNode(int columnIndex, ATypeTag typeTag, boolean primaryKey) {
+ this.columnIndex = columnIndex;
+ this.typeTag = typeTag;
+ this.primaryKey = primaryKey;
+ }
+
+ public PrimitiveSchemaNode(ATypeTag typeTag, DataInput input) throws IOException {
+ this.typeTag = typeTag;
+ columnIndex = input.readInt();
+ primaryKey = input.readBoolean();
+ }
+
+ public final int getColumnIndex() {
+ return columnIndex;
+ }
+
+ @Override
+ public final ATypeTag getTypeTag() {
+ return typeTag;
+ }
+
+ @Override
+ public final boolean isNested() {
+ return false;
+ }
+
+ @Override
+ public final boolean isObjectOrCollection() {
+ return false;
+ }
+
+ @Override
+ public final boolean isCollection() {
+ return false;
+ }
+
+ public final boolean isPrimaryKey() {
+ return primaryKey;
+ }
+
+ @Override
+ public final <R, T> R accept(ISchemaNodeVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ public void serialize(DataOutput output, PathInfoSerializer pathInfoSerializer) throws IOException {
+ output.write(typeTag.serialize());
+ output.writeInt(columnIndex);
+ output.writeBoolean(primaryKey);
+ pathInfoSerializer.writePathInfo(typeTag, columnIndex, primaryKey);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
new file mode 100644
index 0000000..2917074
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.visitor;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ISchemaNodeVisitor;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.MissingFieldSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class PathExtractorVisitor implements ISchemaNodeVisitor<AbstractSchemaNode, Void> {
+ @Override
+ public AbstractSchemaNode visit(ObjectSchemaNode objectNode, Void arg) throws HyracksDataException {
+ int fieldNameIndex = objectNode.getChildrenFieldNameIndexes().getInt(0);
+ if (fieldNameIndex < 0) {
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+ return objectNode.getChild(fieldNameIndex).accept(this, null);
+ }
+
+ @Override
+ public AbstractSchemaNode visit(AbstractCollectionSchemaNode collectionNode, Void arg) throws HyracksDataException {
+ AbstractSchemaNode itemNode = collectionNode.getItemNode();
+ if (itemNode == null) {
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+ return collectionNode.getItemNode().accept(this, null);
+ }
+
+ @Override
+ public AbstractSchemaNode visit(UnionSchemaNode unionNode, Void arg) throws HyracksDataException {
+ for (AbstractSchemaNode node : unionNode.getChildren().values()) {
+ // Using 'for-loop' is the only get the child out of a collection
+ return node.accept(this, null);
+ }
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+
+ @Override
+ public AbstractSchemaNode visit(PrimitiveSchemaNode primitiveNode, Void arg) throws HyracksDataException {
+ //Missing column index is -1
+ return primitiveNode;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaBuilderFromIATypeVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaBuilderFromIATypeVisitor.java
new file mode 100644
index 0000000..fb098fa
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaBuilderFromIATypeVisitor.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.visitor;
+
+import java.util.List;
+
+import org.apache.asterix.column.metadata.FieldNamesDictionary;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.ArraySchemaNode;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public class SchemaBuilderFromIATypeVisitor implements IATypeVisitor<Void, AbstractSchemaNode> {
+ private final FlushColumnMetadata columnMetadata;
+ private final List<List<String>> primaryKeys;
+ private List<String> currentPrimaryKeyPath;
+ private int processedPrimaryKeys;
+ private int currentPathIndex;
+
+ public SchemaBuilderFromIATypeVisitor(FlushColumnMetadata columnMetadata, List<List<String>> primaryKeys) {
+ this.columnMetadata = columnMetadata;
+ this.primaryKeys = primaryKeys;
+ processedPrimaryKeys = 0;
+ }
+
+ @Override
+ public Void visit(ARecordType recordType, AbstractSchemaNode arg) {
+ ObjectSchemaNode objectNode = (ObjectSchemaNode) arg;
+ columnMetadata.enterLevel(objectNode);
+ try {
+ if (processedPrimaryKeys < primaryKeys.size()) {
+ processPrimaryKeys(recordType, objectNode);
+ }
+ for (int i = 0; i < recordType.getFieldTypes().length; i++) {
+ processField(i, recordType, objectNode);
+ }
+ } catch (HyracksDataException e) {
+ throw new IllegalStateException(e);
+ }
+ columnMetadata.exitLevel(objectNode);
+ return null;
+ }
+
+ @Override
+ public Void visit(AbstractCollectionType collectionType, AbstractSchemaNode arg) {
+ ArraySchemaNode collectionNode = (ArraySchemaNode) arg;
+ IAType itemType = collectionType.getItemType();
+ columnMetadata.enterLevel(collectionNode);
+ try {
+ AbstractSchemaNode itemNode = collectionNode.getOrCreateItem(itemType.getTypeTag(), columnMetadata);
+ itemType.accept(this, itemNode);
+ } catch (HyracksDataException e) {
+ throw new IllegalStateException(e);
+ }
+ columnMetadata.exitLevel(collectionNode);
+ return null;
+ }
+
+ @Override
+ public Void visit(AUnionType unionType, AbstractSchemaNode arg) {
+ throw new IllegalStateException(unionType.getTypeTag() + " is not a declared type");
+ }
+
+ @Override
+ public Void visitFlat(IAType flatType, AbstractSchemaNode arg) {
+ if (processedPrimaryKeys < primaryKeys.size()) {
+ processedPrimaryKeys++;
+ }
+ return null;
+ }
+
+ /*
+ * **************************************************************
+ * Handling primary keys and record fields conversion
+ * **************************************************************
+ */
+ private void processPrimaryKeys(ARecordType recordType, ObjectSchemaNode objectNode) throws HyracksDataException {
+ if (objectNode == columnMetadata.getRoot() || objectNode == columnMetadata.getMetaRoot()) {
+ while (processedPrimaryKeys < primaryKeys.size()) {
+ currentPrimaryKeyPath = primaryKeys.get(processedPrimaryKeys);
+ currentPathIndex = 0;
+ processPrimaryKeyPath(recordType, objectNode);
+ }
+ } else {
+ currentPathIndex++;
+ processPrimaryKeyPath(recordType, objectNode);
+ }
+ }
+
+ private void processPrimaryKeyPath(ARecordType recordType, ObjectSchemaNode objectNode)
+ throws HyracksDataException {
+ int fieldIndex = recordType.getFieldIndex(currentPrimaryKeyPath.get(currentPathIndex));
+ processField(fieldIndex, recordType, objectNode);
+ }
+
+ private void processField(int fieldIndex, ARecordType recordType, ObjectSchemaNode objectNode)
+ throws HyracksDataException {
+ IAType[] fieldTypes = recordType.getFieldTypes();
+ String[] fieldNames = recordType.getFieldNames();
+ FieldNamesDictionary dictionary = columnMetadata.getFieldNamesDictionary();
+
+ int fieldNameIndex = dictionary.getOrCreateFieldNameIndex(fieldNames[fieldIndex]);
+ IValueReference fieldName = dictionary.getFieldName(fieldNameIndex);
+
+ IAType fieldType = fieldTypes[fieldIndex];
+ AbstractSchemaNode child = objectNode.getOrCreateChild(fieldName, fieldType.getTypeTag(), columnMetadata);
+
+ fieldType.accept(this, child);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaClipperVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaClipperVisitor.java
new file mode 100644
index 0000000..0771ccb
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaClipperVisitor.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.visitor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.FieldNamesDictionary;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.MissingFieldSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.Warning;
+
+public class SchemaClipperVisitor implements IATypeVisitor<AbstractSchemaNode, AbstractSchemaNode> {
+ private final FieldNamesDictionary fieldNamesDictionary;
+ private final IWarningCollector warningCollector;
+ private final Map<String, FunctionCallInformation> functionCallInfoMap;
+
+ public SchemaClipperVisitor(FieldNamesDictionary fieldNamesDictionary,
+ Map<String, FunctionCallInformation> functionCallInfoMap, IWarningCollector warningCollector) {
+ this.fieldNamesDictionary = fieldNamesDictionary;
+ this.functionCallInfoMap = functionCallInfoMap;
+ this.warningCollector = warningCollector;
+ }
+
+ @Override
+ public AbstractSchemaNode visit(ARecordType recordType, AbstractSchemaNode arg) {
+ if (isNotCompatible(recordType, arg)) {
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+
+ String[] fieldNames = recordType.getFieldNames();
+ IAType[] fieldTypes = recordType.getFieldTypes();
+ ObjectSchemaNode objectNode = getActualNode(arg, ATypeTag.OBJECT, ObjectSchemaNode.class);
+
+ ObjectSchemaNode clippedObjectNode = new ObjectSchemaNode();
+ try {
+ for (int i = 0; i < fieldNames.length; i++) {
+ int fieldNameIndex = fieldNamesDictionary.getFieldNameIndex(fieldNames[i]);
+ AbstractSchemaNode child = objectNode.getChild(fieldNameIndex);
+ clippedObjectNode.addChild(fieldNameIndex, fieldTypes[i].accept(this, child));
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+
+ return clippedObjectNode;
+ }
+
+ @Override
+ public AbstractSchemaNode visit(AbstractCollectionType collectionType, AbstractSchemaNode arg) {
+ if (isNotCompatible(collectionType, arg)) {
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+ AbstractCollectionSchemaNode collectionNode =
+ getActualNode(arg, collectionType.getTypeTag(), AbstractCollectionSchemaNode.class);
+ AbstractSchemaNode newItemNode = collectionType.getItemType().accept(this, collectionNode.getItemNode());
+ AbstractCollectionSchemaNode clippedCollectionNode =
+ AbstractCollectionSchemaNode.create(collectionType.getTypeTag());
+ clippedCollectionNode.setItemNode(newItemNode);
+ return clippedCollectionNode;
+ }
+
+ @Override
+ public AbstractSchemaNode visit(AUnionType unionType, AbstractSchemaNode arg) {
+ return arg;
+ }
+
+ @Override
+ public AbstractSchemaNode visitFlat(IAType flatType, AbstractSchemaNode arg) {
+ if (flatType.getTypeTag() == ATypeTag.ANY) {
+ return arg;
+ } else if (isNotCompatible(flatType, arg)) {
+ return getNonCompatibleNumericNodeIfAny(flatType, arg);
+ }
+ return getActualNode(arg, flatType.getTypeTag(), PrimitiveSchemaNode.class);
+ }
+
+ private AbstractSchemaNode getNonCompatibleNumericNodeIfAny(IAType flatType, AbstractSchemaNode arg) {
+ ATypeHierarchy.Domain requestedDomain = ATypeHierarchy.getTypeDomain(flatType.getTypeTag());
+ ATypeHierarchy.Domain nodeDomain = ATypeHierarchy.getTypeDomain(arg.getTypeTag());
+ if (nodeDomain == requestedDomain && nodeDomain == ATypeHierarchy.Domain.NUMERIC) {
+ // This will be reconciled by the filter accessor
+ return arg;
+ } else if (arg.getTypeTag() == ATypeTag.UNION) {
+ UnionSchemaNode unionNode = (UnionSchemaNode) arg;
+ return unionNode.getNumericChildOrMissing();
+ }
+
+ return MissingFieldSchemaNode.INSTANCE;
+ }
+
+ private <T extends AbstractSchemaNode> T getActualNode(AbstractSchemaNode node, ATypeTag typeTag, Class<T> clazz) {
+ if (node.getTypeTag() == typeTag) {
+ return clazz.cast(node);
+ } else {
+ //Then it is a union (as we check for incompatibility before we call this method)
+ UnionSchemaNode unionNode = (UnionSchemaNode) node;
+ return clazz.cast(unionNode.getChild(typeTag));
+ }
+ }
+
+ private boolean isNotCompatible(IAType requestedType, AbstractSchemaNode schemaNode) {
+ if (requestedType.getTypeTag() != schemaNode.getTypeTag()) {
+ if (schemaNode.getTypeTag() != ATypeTag.UNION) {
+ warn(requestedType, schemaNode);
+ return true;
+ }
+ // Handle union
+ UnionSchemaNode unionNode = (UnionSchemaNode) schemaNode;
+ return notInUnion(requestedType, unionNode);
+ }
+ return unionContainsMultipleNumeric(schemaNode);
+ }
+
+ private boolean notInUnion(IAType requestedType, UnionSchemaNode unionNode) {
+ for (AbstractSchemaNode unionChildNode : unionNode.getChildren().values()) {
+ warn(requestedType, unionChildNode);
+ }
+ return !unionNode.getChildren().containsKey(requestedType.getTypeTag());
+ }
+
+ private void warn(IAType requestedType, AbstractSchemaNode schemaNode) {
+ if (ATypeHierarchy.isCompatible(requestedType.getTypeTag(), schemaNode.getTypeTag())) {
+ return;
+ }
+ if (warningCollector.shouldWarn()) {
+ Warning warning = functionCallInfoMap.get(requestedType.getTypeName())
+ .createWarning(requestedType.getTypeTag(), schemaNode.getTypeTag());
+ if (warning != null) {
+ warningCollector.warn(warning);
+ }
+ }
+ }
+
+ private boolean unionContainsMultipleNumeric(AbstractSchemaNode schemaNode) {
+ if (schemaNode.getTypeTag() == ATypeTag.UNION) {
+ UnionSchemaNode unionNode = (UnionSchemaNode) schemaNode;
+ return unionNode.getNumberOfNumericChildren() > 1;
+ }
+ return false;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java
new file mode 100644
index 0000000..4cbe09b
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.flush;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ISchemaNodeVisitor;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.column.values.IColumnBatchWriter;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public final class BatchFinalizerVisitor implements ISchemaNodeVisitor<Void, AbstractSchemaNestedNode> {
+ private final FlushColumnMetadata columnSchemaMetadata;
+ private final IColumnValuesWriter[] primaryKeyWriters;
+ private final PriorityQueue<IColumnValuesWriter> orderedColumns;
+ private int level;
+
+ public BatchFinalizerVisitor(FlushColumnMetadata columnSchemaMetadata) {
+ this.columnSchemaMetadata = columnSchemaMetadata;
+ orderedColumns = new PriorityQueue<>(Comparator.comparingInt(x -> -x.getEstimatedSize()));
+ int numberOfPrimaryKeys = columnSchemaMetadata.getNumberOfPrimaryKeys();
+ primaryKeyWriters = new IColumnValuesWriter[numberOfPrimaryKeys];
+ for (int i = 0; i < numberOfPrimaryKeys; i++) {
+ primaryKeyWriters[i] = columnSchemaMetadata.getWriter(i);
+ }
+ level = -1;
+ }
+
+ public int finalizeBatch(IColumnBatchWriter batchWriter, FlushColumnMetadata columnMetadata)
+ throws HyracksDataException {
+ orderedColumns.clear();
+
+ columnMetadata.getRoot().accept(this, null);
+ if (columnMetadata.getMetaRoot() != null) {
+ columnMetadata.getMetaRoot().accept(this, null);
+ }
+
+ int allocatedSpace = batchWriter.writePrimaryKeyColumns(primaryKeyWriters);
+ allocatedSpace += batchWriter.writeColumns(orderedColumns);
+ return allocatedSpace;
+ }
+
+ @Override
+ public Void visit(ObjectSchemaNode objectNode, AbstractSchemaNestedNode arg) throws HyracksDataException {
+ level++;
+ columnSchemaMetadata.flushDefinitionLevels(level, arg, objectNode);
+ List<AbstractSchemaNode> children = objectNode.getChildren();
+ for (int i = 0; i < children.size(); i++) {
+ children.get(i).accept(this, objectNode);
+ }
+ objectNode.setCounter(0);
+ columnSchemaMetadata.clearDefinitionLevels(objectNode);
+ level--;
+ return null;
+ }
+
+ @Override
+ public Void visit(AbstractCollectionSchemaNode collectionNode, AbstractSchemaNestedNode arg)
+ throws HyracksDataException {
+ level++;
+ columnSchemaMetadata.flushDefinitionLevels(level, arg, collectionNode);
+ collectionNode.getItemNode().accept(this, collectionNode);
+ collectionNode.setCounter(0);
+ columnSchemaMetadata.clearDefinitionLevels(collectionNode);
+ level--;
+ return null;
+ }
+
+ @Override
+ public Void visit(UnionSchemaNode unionNode, AbstractSchemaNestedNode arg) throws HyracksDataException {
+ columnSchemaMetadata.flushDefinitionLevels(level, arg, unionNode);
+ for (AbstractSchemaNode node : unionNode.getChildren().values()) {
+ node.accept(this, unionNode);
+ }
+ unionNode.setCounter(0);
+ columnSchemaMetadata.clearDefinitionLevels(unionNode);
+ return null;
+ }
+
+ @Override
+ public Void visit(PrimitiveSchemaNode primitiveNode, AbstractSchemaNestedNode arg) throws HyracksDataException {
+ columnSchemaMetadata.flushDefinitionLevels(level, arg, primitiveNode);
+ if (!primitiveNode.isPrimaryKey()) {
+ orderedColumns.add(columnSchemaMetadata.getWriter(primitiveNode.getColumnIndex()));
+ }
+
+ //Prepare for the next batch
+ primitiveNode.setCounter(0);
+ return null;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java
new file mode 100644
index 0000000..48cd442
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.flush;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.om.lazy.AbstractLazyVisitablePointable;
+import org.apache.asterix.om.lazy.AbstractListLazyVisitablePointable;
+import org.apache.asterix.om.lazy.FlatLazyVisitablePointable;
+import org.apache.asterix.om.lazy.ILazyVisitablePointableVisitor;
+import org.apache.asterix.om.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
+
+public class ColumnTransformer implements ILazyVisitablePointableVisitor<AbstractSchemaNode, AbstractSchemaNode> {
+ private final FlushColumnMetadata columnMetadata;
+ private final VoidPointable nonTaggedValue;
+ private final ObjectSchemaNode root;
+ private AbstractSchemaNestedNode currentParent;
+ private int primaryKeysLength;
+
+ public ColumnTransformer(FlushColumnMetadata columnMetadata, ObjectSchemaNode root) {
+ this.columnMetadata = columnMetadata;
+ this.root = root;
+ nonTaggedValue = new VoidPointable();
+ }
+
+ /**
+ * Transform a tuple in row format into columns
+ *
+ * @param pointable record pointable
+ * @return the estimated size (possibly overestimated) of the primary key(s) columns
+ */
+ public int transform(RecordLazyVisitablePointable pointable) throws HyracksDataException {
+ primaryKeysLength = 0;
+ pointable.accept(this, root);
+ return primaryKeysLength;
+ }
+
+ public int writeAntiMatter(LSMBTreeTupleReference tuple) throws HyracksDataException {
+ int pkSize = 0;
+ for (int i = 0; i < columnMetadata.getNumberOfPrimaryKeys(); i++) {
+ byte[] bytes = tuple.getFieldData(i);
+ int start = tuple.getFieldStart(i);
+ ATypeTag tag = ATypeTag.VALUE_TYPE_MAPPING[bytes[start]];
+ nonTaggedValue.set(bytes, start + 1, tuple.getFieldLength(i) - 1);
+ IColumnValuesWriter writer = columnMetadata.getWriter(i);
+ writer.writeAntiMatter(tag, nonTaggedValue);
+ pkSize += writer.getEstimatedSize();
+ }
+ return pkSize;
+ }
+
+ @Override
+ public AbstractSchemaNode visit(RecordLazyVisitablePointable pointable, AbstractSchemaNode arg)
+ throws HyracksDataException {
+ columnMetadata.enterNode(currentParent, arg);
+ AbstractSchemaNestedNode previousParent = currentParent;
+
+ ObjectSchemaNode objectNode = (ObjectSchemaNode) arg;
+ currentParent = objectNode;
+ for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
+ pointable.nextChild();
+ IValueReference fieldName = pointable.getFieldName();
+ ATypeTag childTypeTag = pointable.getChildTypeTag();
+ if (childTypeTag != ATypeTag.MISSING) {
+ //Only write actual field values (including NULL) but ignore MISSING fields
+ AbstractSchemaNode childNode = objectNode.getOrCreateChild(fieldName, childTypeTag, columnMetadata);
+ acceptActualNode(pointable.getChildVisitablePointable(), childNode);
+ }
+ }
+
+ columnMetadata.exitNode(arg);
+ currentParent = previousParent;
+ return null;
+ }
+
+ @Override
+ public AbstractSchemaNode visit(AbstractListLazyVisitablePointable pointable, AbstractSchemaNode arg)
+ throws HyracksDataException {
+ columnMetadata.enterNode(currentParent, arg);
+ AbstractSchemaNestedNode previousParent = currentParent;
+
+ AbstractCollectionSchemaNode collectionNode = (AbstractCollectionSchemaNode) arg;
+ RunLengthIntArray defLevels = columnMetadata.getDefinitionLevels(collectionNode);
+ //the level at which an item is missing
+ int missingLevel = columnMetadata.getLevel();
+ currentParent = collectionNode;
+
+ int numberOfChildren = pointable.getNumberOfChildren();
+ for (int i = 0; i < numberOfChildren; i++) {
+ pointable.nextChild();
+ ATypeTag childTypeTag = pointable.getChildTypeTag();
+ AbstractSchemaNode childNode = collectionNode.getOrCreateItem(childTypeTag, columnMetadata);
+ acceptActualNode(pointable.getChildVisitablePointable(), childNode);
+ /*
+ * The array item may change (e.g., BIGINT --> UNION). Thus, new items would be considered as missing
+ */
+ defLevels.add(missingLevel);
+ }
+
+ columnMetadata.exitCollectionNode(collectionNode, numberOfChildren);
+ currentParent = previousParent;
+ return null;
+ }
+
+ @Override
+ public AbstractSchemaNode visit(FlatLazyVisitablePointable pointable, AbstractSchemaNode arg)
+ throws HyracksDataException {
+ columnMetadata.enterNode(currentParent, arg);
+ ATypeTag valueTypeTag = pointable.getTypeTag();
+ PrimitiveSchemaNode node = (PrimitiveSchemaNode) arg;
+ IColumnValuesWriter writer = columnMetadata.getWriter(node.getColumnIndex());
+ if (valueTypeTag == ATypeTag.MISSING) {
+ writer.writeLevel(columnMetadata.getLevel());
+ } else if (valueTypeTag == ATypeTag.NULL) {
+ writer.writeNull(columnMetadata.getLevel());
+ } else if (pointable.isTagged()) {
+ //Remove type tag
+ nonTaggedValue.set(pointable.getByteArray(), pointable.getStartOffset() + 1, pointable.getLength() - 1);
+ writer.writeValue(pointable.getTypeTag(), nonTaggedValue);
+ } else {
+ writer.writeValue(pointable.getTypeTag(), pointable);
+ }
+ if (node.isPrimaryKey()) {
+ primaryKeysLength += writer.getEstimatedSize();
+ }
+ columnMetadata.exitNode(arg);
+ return null;
+ }
+
+ private void acceptActualNode(AbstractLazyVisitablePointable pointable, AbstractSchemaNode node)
+ throws HyracksDataException {
+ if (node.getTypeTag() == ATypeTag.UNION) {
+ columnMetadata.enterNode(currentParent, node);
+ AbstractSchemaNestedNode previousParent = currentParent;
+
+ UnionSchemaNode unionNode = (UnionSchemaNode) node;
+ currentParent = unionNode;
+
+ ATypeTag childTypeTag = pointable.getTypeTag();
+ AbstractSchemaNode actualNode;
+ if (childTypeTag == ATypeTag.NULL || childTypeTag == ATypeTag.MISSING) {
+ actualNode = unionNode.getOriginalType();
+ } else {
+ actualNode = unionNode.getOrCreateChild(pointable.getTypeTag(), columnMetadata);
+ }
+ pointable.accept(this, actualNode);
+
+ currentParent = previousParent;
+ columnMetadata.exitNode(node);
+ } else if (pointable.getTypeTag() == ATypeTag.NULL && node.isNested()) {
+ columnMetadata.addNestedNull(currentParent, (AbstractSchemaNestedNode) node);
+ } else {
+ pointable.accept(this, node);
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
new file mode 100644
index 0000000..8cd1e98
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
@@ -0,0 +1,572 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.flush;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.AbstractColumnMetadata;
+import org.apache.asterix.column.metadata.FieldNamesDictionary;
+import org.apache.asterix.column.metadata.PathInfoSerializer;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.ArraySchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.MultisetSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.column.metadata.schema.visitor.SchemaBuilderFromIATypeVisitor;
+import org.apache.asterix.column.util.ColumnValuesUtil;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.IColumnValuesWriterFactory;
+import org.apache.asterix.column.values.writer.AbstractColumnValuesWriter;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+
+/**
+ * Flush column metadata belongs to a flushing {@link ILSMMemoryComponent}
+ * The schema here is mutable and can change according to the flushed records
+ */
+public final class FlushColumnMetadata extends AbstractColumnMetadata {
+ private final Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels;
+ private final Mutable<IColumnWriteMultiPageOp> multiPageOpRef;
+ private final FieldNamesDictionary fieldNamesDictionary;
+ private final ObjectSchemaNode root;
+ private final ObjectSchemaNode metaRoot;
+ private final IColumnValuesWriterFactory columnWriterFactory;
+ private final List<IColumnValuesWriter> columnWriters;
+ private final ArrayBackedValueStorage serializedMetadata;
+ private final PathInfoSerializer pathInfoSerializer;
+ private final IntArrayList nullWriterIndexes;
+ private final boolean metaContainsKeys;
+ private boolean changed;
+ private int level;
+ private int repeated;
+
+ public FlushColumnMetadata(ARecordType datasetType, ARecordType metaType, List<List<String>> primaryKeys,
+ List<Integer> keySourceIndicator, IColumnValuesWriterFactory columnWriterFactory,
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef) throws HyracksDataException {
+ super(datasetType, metaType, primaryKeys.size());
+ this.multiPageOpRef = multiPageOpRef;
+ this.columnWriterFactory = columnWriterFactory;
+ definitionLevels = new HashMap<>();
+ columnWriters = new ArrayList<>();
+ level = -1;
+ repeated = 0;
+ fieldNamesDictionary = new FieldNamesDictionary();
+ root = new ObjectSchemaNode();
+ metaRoot = metaType != null ? new ObjectSchemaNode() : null;
+ pathInfoSerializer = new PathInfoSerializer();
+ nullWriterIndexes = new IntArrayList();
+ //Add definition levels for the root
+ addDefinitionLevelsAndGet(root);
+ SchemaBuilderFromIATypeVisitor builder = new SchemaBuilderFromIATypeVisitor(this, primaryKeys);
+ //Ensure all primary keys take the first column indexes
+ metaContainsKeys = metaType != null && keySourceIndicator.get(0) == 1;
+ if (metaContainsKeys) {
+ addDefinitionLevelsAndGet(metaRoot);
+ metaType.accept(builder, metaRoot);
+ datasetType.accept(builder, root);
+ } else {
+ datasetType.accept(builder, root);
+ if (metaRoot != null) {
+ addDefinitionLevelsAndGet(metaRoot);
+ metaType.accept(builder, metaRoot);
+ }
+ }
+
+ serializedMetadata = new ArrayBackedValueStorage();
+ changed = true;
+ serializeColumnsMetadata();
+ }
+
+ private FlushColumnMetadata(ARecordType datasetType, ARecordType metaType, List<List<String>> primaryKeys,
+ boolean metaContainsKeys, IColumnValuesWriterFactory columnWriterFactory,
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef, List<IColumnValuesWriter> columnWriters,
+ FieldNamesDictionary fieldNamesDictionary, ObjectSchemaNode root, ObjectSchemaNode metaRoot,
+ Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels,
+ ArrayBackedValueStorage serializedMetadata) {
+ super(datasetType, metaType, primaryKeys.size());
+ this.multiPageOpRef = multiPageOpRef;
+ this.columnWriterFactory = columnWriterFactory;
+ this.definitionLevels = definitionLevels;
+ this.columnWriters = columnWriters;
+ level = -1;
+ repeated = 0;
+ this.fieldNamesDictionary = fieldNamesDictionary;
+ this.root = root;
+ this.metaRoot = metaRoot;
+ this.metaContainsKeys = metaContainsKeys;
+ pathInfoSerializer = new PathInfoSerializer();
+ nullWriterIndexes = new IntArrayList();
+ //Add definition levels for the root
+ addDefinitionLevelsAndGet(root);
+ this.serializedMetadata = serializedMetadata;
+ changed = false;
+ }
+
+ public FieldNamesDictionary getFieldNamesDictionary() {
+ return fieldNamesDictionary;
+ }
+
+ public ObjectSchemaNode getRoot() {
+ return root;
+ }
+
+ public ObjectSchemaNode getMetaRoot() {
+ return metaRoot;
+ }
+
+ public Mutable<IColumnWriteMultiPageOp> getMultiPageOpRef() {
+ return multiPageOpRef;
+ }
+
+ @Override
+ public IValueReference serializeColumnsMetadata() throws HyracksDataException {
+ if (changed) {
+ try {
+ serializeChanges();
+ changed = false;
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ return serializedMetadata;
+ }
+
+ private void serializeChanges() throws IOException {
+ serializedMetadata.reset();
+ DataOutput output = serializedMetadata.getDataOutput();
+
+ int writersOffsetPointer = reserveInt(output);
+ int fieldNamesOffsetPointer = reserveInt(output);
+ int schemaOffsetPointer = reserveInt(output);
+ int metaSchemaOffsetPointer = reserveInt(output);
+ int pathInfoOffsetPointer = reserveInt(output);
+
+ //ColumnWriterInformation
+ setOffset(writersOffsetPointer);
+ output.writeInt(columnWriters.size());
+ for (IColumnValuesWriter writer : columnWriters) {
+ writer.serialize(output);
+ }
+
+ //FieldNames
+ setOffset(fieldNamesOffsetPointer);
+ fieldNamesDictionary.serialize(output);
+
+ //Schema
+ pathInfoSerializer.reset();
+ setOffset(schemaOffsetPointer);
+ root.serialize(output, pathInfoSerializer);
+ if (metaRoot != null) {
+ //Meta schema
+ setOffset(metaSchemaOffsetPointer);
+ metaRoot.serialize(output, pathInfoSerializer);
+ }
+
+ //Path info
+ setOffset(pathInfoOffsetPointer);
+ pathInfoSerializer.serialize(output, getNumberOfColumns());
+ }
+
+ private int reserveInt(DataOutput output) throws IOException {
+ int offset = serializedMetadata.getLength();
+ output.writeInt(-1);
+ return offset;
+ }
+
+ private void setOffset(int pointer) {
+ int offset = serializedMetadata.getLength();
+ IntegerPointable.setInteger(serializedMetadata.getByteArray(), pointer, offset);
+ }
+
+ public static FlushColumnMetadata create(ARecordType datasetType, ARecordType metaType,
+ List<List<String>> primaryKeys, List<Integer> keySourceIndicator,
+ IColumnValuesWriterFactory columnWriterFactory, Mutable<IColumnWriteMultiPageOp> multiPageOpRef,
+ IValueReference serializedMetadata) throws HyracksDataException {
+ boolean metaContainsKeys = metaType != null && keySourceIndicator.get(0) == 1;
+ try {
+ return createMutableMetadata(datasetType, metaType, primaryKeys, metaContainsKeys, columnWriterFactory,
+ multiPageOpRef, serializedMetadata);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private static FlushColumnMetadata createMutableMetadata(ARecordType datasetType, ARecordType metaType,
+ List<List<String>> primaryKeys, boolean metaContainsKeys, IColumnValuesWriterFactory columnWriterFactory,
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef, IValueReference serializedMetadata) throws IOException {
+ DataInput input = new DataInputStream(new ByteArrayInputStream(serializedMetadata.getByteArray(),
+ serializedMetadata.getStartOffset(), serializedMetadata.getLength()));
+ //Skip offsets
+ input.skipBytes(OFFSETS_SIZE);
+
+ //ColumnWriter
+ List<IColumnValuesWriter> writers = new ArrayList<>();
+ deserializeWriters(input, writers, columnWriterFactory);
+
+ //FieldNames
+ FieldNamesDictionary fieldNamesDictionary = FieldNamesDictionary.deserialize(input);
+
+ //Schema
+ Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels = new HashMap<>();
+ ObjectSchemaNode root = (ObjectSchemaNode) AbstractSchemaNode.deserialize(input, definitionLevels);
+ ObjectSchemaNode metaRoot = null;
+ if (metaType != null) {
+ metaRoot = (ObjectSchemaNode) AbstractSchemaNode.deserialize(input, definitionLevels);
+ }
+
+ ArrayBackedValueStorage schemaStorage = new ArrayBackedValueStorage(serializedMetadata.getLength());
+ schemaStorage.append(serializedMetadata);
+ return new FlushColumnMetadata(datasetType, metaType, primaryKeys, metaContainsKeys, columnWriterFactory,
+ multiPageOpRef, writers, fieldNamesDictionary, root, metaRoot, definitionLevels, schemaStorage);
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ DataInputStream input = new DataInputStream(new ByteArrayInputStream(serializedMetadata.getByteArray()));
+ try {
+ abort(input);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private void abort(DataInputStream input) throws IOException {
+ level = -1;
+ repeated = 0;
+ changed = false;
+
+ columnWriters.clear();
+ deserializeWriters(input, columnWriters, columnWriterFactory);
+
+ fieldNamesDictionary.abort(input);
+ definitionLevels.clear();
+ root.abort(input, definitionLevels);
+ }
+
+ public static void deserializeWriters(DataInput input, List<IColumnValuesWriter> writers,
+ IColumnValuesWriterFactory columnWriterFactory) throws IOException {
+ int numberOfWriters = input.readInt();
+ for (int i = 0; i < numberOfWriters; i++) {
+ writers.add(AbstractColumnValuesWriter.deserialize(input, columnWriterFactory));
+ }
+ }
+
+ /* ********************************************************
+ * Column values related methods
+ * ********************************************************
+ */
+
+ /**
+ * Set {@link IColumnWriteMultiPageOp} for {@link IColumnValuesWriter}
+ *
+ * @param multiPageOp multi-buffer allocator
+ */
+ public void init(IColumnWriteMultiPageOp multiPageOp) throws HyracksDataException {
+ multiPageOpRef.setValue(multiPageOp);
+
+ //Reset writer for the first write
+ for (int i = 0; i < columnWriters.size(); i++) {
+ columnWriters.get(i).reset();
+ }
+ }
+
+ public IColumnValuesWriter getWriter(int columnIndex) {
+ return columnWriters.get(columnIndex);
+ }
+
+ /* ********************************************************
+ * Schema related methods
+ * ********************************************************
+ */
+
+ public int getLevel() {
+ return level;
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return columnWriters.size();
+ }
+
+ public AbstractSchemaNode getOrCreateChild(AbstractSchemaNode child, ATypeTag childTypeTag)
+ throws HyracksDataException {
+ AbstractSchemaNode currentChild = child;
+ ATypeTag normalizedTypeTag = getNormalizedTypeTag(childTypeTag);
+ if (currentChild == null || normalizedTypeTag != ATypeTag.MISSING && normalizedTypeTag != ATypeTag.NULL
+ && currentChild.getTypeTag() != ATypeTag.UNION && currentChild.getTypeTag() != normalizedTypeTag) {
+ //Create a new child or union type if required type is different from the current child type
+ currentChild = createChild(child, normalizedTypeTag);
+ //Flag that the schema has changed
+ changed = true;
+ }
+ return currentChild;
+ }
+
+ public void enterLevel(AbstractSchemaNestedNode node) {
+ level++;
+ if (node.isCollection()) {
+ repeated++;
+ }
+ }
+
+ public void exitLevel(AbstractSchemaNestedNode node) {
+ level--;
+ if (node.isCollection()) {
+ repeated--;
+ }
+ }
+
+ public void enterNode(AbstractSchemaNestedNode parent, AbstractSchemaNode node) throws HyracksDataException {
+ //Flush all definition levels from parent to child
+ flushDefinitionLevels(level, parent, node);
+ if (node.isObjectOrCollection()) {
+ //Enter one more level for object, array, and multiset
+ level++;
+ if (node.isCollection()) {
+ //Tells nested values that they are repeated
+ repeated++;
+ }
+ }
+ }
+
+ public void exitNode(AbstractSchemaNode node) {
+ if (node.isNested()) {
+ //Add the nested node's level for all missing children (i.e., not entered for a record)
+ definitionLevels.get((AbstractSchemaNestedNode) node).add(level);
+ if (node.isObjectOrCollection()) {
+ //Union nodes should not change the level as they are logical nodes
+ level--;
+ }
+ }
+ node.incrementCounter();
+ }
+
+ public void exitCollectionNode(AbstractCollectionSchemaNode collectionNode, int numberOfItems) {
+ RunLengthIntArray collectionDefLevels = definitionLevels.get(collectionNode);
+ //Add delimiter
+ collectionDefLevels.add(level - 1);
+ level--;
+ repeated--;
+ collectionNode.incrementCounter();
+ }
+
+ /**
+ * Needed by {@link AbstractCollectionSchemaNode} to add the definition level for each item
+ *
+ * @param collectionSchemaNode collection node
+ * @return collection node's definition level
+ */
+ public RunLengthIntArray getDefinitionLevels(AbstractCollectionSchemaNode collectionSchemaNode) {
+ return definitionLevels.get(collectionSchemaNode);
+ }
+
+ public void clearDefinitionLevels(AbstractSchemaNestedNode nestedNode) {
+ definitionLevels.get(nestedNode).reset();
+ }
+
+ public void flushDefinitionLevels(int level, AbstractSchemaNestedNode parent, AbstractSchemaNode node)
+ throws HyracksDataException {
+ if (parent != null) {
+ RunLengthIntArray parentDefLevels = definitionLevels.get(parent);
+ if (node.getCounter() < parentDefLevels.getSize()) {
+ int parentMask = ColumnValuesUtil.getNullMask(level);
+ int childMask = ColumnValuesUtil.getNullMask(level + 1);
+ flushDefinitionLevels(parentMask, childMask, parentDefLevels, node);
+ }
+ }
+ }
+
+ private void flushDefinitionLevels(int parentMask, int childMask, RunLengthIntArray parentDefLevels,
+ AbstractSchemaNode node) throws HyracksDataException {
+ int startIndex = node.getCounter();
+ if (node.isNested()) {
+ RunLengthIntArray childDefLevels = definitionLevels.get((AbstractSchemaNestedNode) node);
+ flushNestedDefinitionLevel(parentMask, childMask, startIndex, parentDefLevels, childDefLevels);
+ } else {
+ IColumnValuesWriter writer = columnWriters.get(((PrimitiveSchemaNode) node).getColumnIndex());
+ flushWriterDefinitionLevels(parentMask, childMask, startIndex, parentDefLevels, writer);
+ }
+ node.setCounter(parentDefLevels.getSize());
+ }
+
+ private void flushNestedDefinitionLevel(int parentMask, int childMask, int startIndex,
+ RunLengthIntArray parentDefLevels, RunLengthIntArray childDefLevels) {
+ if (parentDefLevels.getSize() == 0) {
+ return;
+ }
+ //First, handle the first block as startIndex might be at the middle of a block
+ //Get which block that startIndex resides
+ int blockIndex = parentDefLevels.getBlockIndex(startIndex);
+ //Get the remaining of the first block starting from startIndex
+ int remainingValues = parentDefLevels.getBlockSize(blockIndex, startIndex);
+
+ int firstBlockValue =
+ ColumnValuesUtil.getChildValue(parentMask, childMask, parentDefLevels.getBlockValue(blockIndex));
+ //Batch add all the remaining values
+ childDefLevels.add(firstBlockValue, remainingValues);
+
+ //Add other blocks as batches
+ for (int i = blockIndex + 1; i < parentDefLevels.getNumberOfBlocks(); i++) {
+ int blockValue = ColumnValuesUtil.getChildValue(parentMask, childMask, parentDefLevels.getBlockValue(i));
+ childDefLevels.add(blockValue, parentDefLevels.getBlockSize(i));
+ }
+ }
+
+ private void flushWriterDefinitionLevels(int parentMask, int childMask, int startIndex,
+ RunLengthIntArray parentDefLevels, IColumnValuesWriter writer) throws HyracksDataException {
+ if (parentDefLevels.getSize() == 0) {
+ return;
+ }
+ /*
+ * We might need only a fraction of the first block. Hence, we first determine how many definition level
+ * values we need. Then, we write those definition levels.
+ */
+ int blockIndex = parentDefLevels.getBlockIndex(startIndex);
+ int remainingValues = parentDefLevels.getBlockSize(blockIndex, startIndex);
+ int firstBlockValue =
+ ColumnValuesUtil.getChildValue(parentMask, childMask, parentDefLevels.getBlockValue(blockIndex));
+ writer.writeLevels(firstBlockValue, remainingValues);
+
+ //Write remaining definition levels from the remaining blocks
+ for (int i = blockIndex + 1; i < parentDefLevels.getNumberOfBlocks(); i++) {
+ int blockValue = ColumnValuesUtil.getChildValue(parentMask, childMask, parentDefLevels.getBlockValue(i));
+ writer.writeLevels(blockValue, parentDefLevels.getBlockSize(i));
+ }
+ }
+
+ private AbstractSchemaNode createChild(AbstractSchemaNode child, ATypeTag normalizedTypeTag)
+ throws HyracksDataException {
+ AbstractSchemaNode createdChild;
+ if (child != null) {
+ if (child.getTypeTag() == ATypeTag.NULL) {
+ //The previous child was a NULL. The new child needs to inherit the NULL definition levels
+ int columnIndex = ((PrimitiveSchemaNode) child).getColumnIndex();
+ RunLengthIntArray defLevels = columnWriters.get(columnIndex).getDefinitionLevelsIntArray();
+ //Add the column index to be garbage collected
+ nullWriterIndexes.add(columnIndex);
+ createdChild = createChild(normalizedTypeTag);
+ int mask = ColumnValuesUtil.getNullMask(level);
+ flushDefinitionLevels(mask, mask, defLevels, createdChild);
+ } else {
+ //Different type. Make union
+ createdChild = addDefinitionLevelsAndGet(new UnionSchemaNode(child, createChild(normalizedTypeTag)));
+ }
+ } else {
+ createdChild = createChild(normalizedTypeTag);
+ }
+ return createdChild;
+ }
+
+ private AbstractSchemaNode createChild(ATypeTag normalizedTypeTag) throws HyracksDataException {
+ switch (normalizedTypeTag) {
+ case OBJECT:
+ return addDefinitionLevelsAndGet(new ObjectSchemaNode());
+ case ARRAY:
+ return addDefinitionLevelsAndGet(new ArraySchemaNode());
+ case MULTISET:
+ return addDefinitionLevelsAndGet(new MultisetSchemaNode());
+ case NULL:
+ case MISSING:
+ case BOOLEAN:
+ case DOUBLE:
+ case BIGINT:
+ case STRING:
+ case UUID:
+ int columnIndex = nullWriterIndexes.isEmpty() ? columnWriters.size() : nullWriterIndexes.removeInt(0);
+ boolean primaryKey = columnIndex < getNumberOfPrimaryKeys();
+ boolean writeAlways = primaryKey || repeated > 0;
+ boolean filtered = !primaryKey;
+ int maxLevel = primaryKey ? 1 : level + 1;
+ IColumnValuesWriter writer = columnWriterFactory.createValueWriter(normalizedTypeTag, columnIndex,
+ maxLevel, writeAlways, filtered);
+ if (multiPageOpRef.getValue() != null) {
+ writer.reset();
+ }
+ addColumn(columnIndex, writer);
+ return new PrimitiveSchemaNode(columnIndex, normalizedTypeTag, primaryKey);
+ default:
+ throw new IllegalStateException("Unsupported type " + normalizedTypeTag);
+
+ }
+ }
+
+ private void addColumn(int index, IColumnValuesWriter writer) {
+ if (index == columnWriters.size()) {
+ columnWriters.add(writer);
+ } else {
+ columnWriters.set(index, writer);
+ }
+ }
+
+ private AbstractSchemaNode addDefinitionLevelsAndGet(AbstractSchemaNestedNode nestedNode) {
+ definitionLevels.put(nestedNode, new RunLengthIntArray());
+ return nestedNode;
+ }
+
+ public static ATypeTag getNormalizedTypeTag(ATypeTag typeTag) {
+ switch (typeTag) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ return ATypeTag.BIGINT;
+ case FLOAT:
+ return ATypeTag.DOUBLE;
+ default:
+ return typeTag;
+ }
+ }
+
+ public void close() {
+ //Dereference multiPageOp
+ multiPageOpRef.setValue(null);
+ for (int i = 0; i < columnWriters.size(); i++) {
+ columnWriters.get(i).close();
+ }
+ }
+
+ public void addNestedNull(AbstractSchemaNestedNode parent, AbstractSchemaNestedNode node)
+ throws HyracksDataException {
+ //Flush all definition levels from parent to the current node
+ flushDefinitionLevels(level, parent, node);
+ //Add null value (+2) to say that both the parent and the child are present
+ definitionLevels.get(node).add(ColumnValuesUtil.getNullMask(level + 2) | level);
+ node.incrementCounter();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleReaderWriterFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleReaderWriterFactory.java
new file mode 100644
index 0000000..9b1b0a2
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleReaderWriterFactory.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.flush;
+
+import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReaderWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+
+public class FlushColumnTupleReaderWriterFactory extends AbstractColumnTupleReaderWriterFactory {
+ private static final long serialVersionUID = -9197679192729634493L;
+
+ public FlushColumnTupleReaderWriterFactory(int pageSize, int maxNumberOfTuples, float tolerance) {
+ super(pageSize, maxNumberOfTuples, tolerance);
+ }
+
+ @Override
+ public AbstractColumnTupleWriter createColumnWriter(IColumnMetadata columnMetadata) {
+ FlushColumnMetadata flushColumnMetadata = (FlushColumnMetadata) columnMetadata;
+ if (flushColumnMetadata.getMetaType() == null) {
+ //no meta
+ return new FlushColumnTupleWriter(flushColumnMetadata, pageSize, maxNumberOfTuples, tolerance);
+ }
+ return new FlushColumnTupleWithMetaWriter(flushColumnMetadata, pageSize, maxNumberOfTuples, tolerance);
+ }
+
+ @Override
+ public AbstractColumnTupleReader createColumnReader(IColumnProjectionInfo columnProjectionInfo) {
+ return ((AbstractColumnImmutableReadMetadata) columnProjectionInfo).createTupleReader();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWithMetaWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWithMetaWriter.java
new file mode 100644
index 0000000..9c527da
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWithMetaWriter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.flush;
+
+import org.apache.asterix.om.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.lazy.TypedRecordLazyVisitablePointable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
+
+public class FlushColumnTupleWithMetaWriter extends FlushColumnTupleWriter {
+ private final ColumnTransformer metaColumnTransformer;
+ private final RecordLazyVisitablePointable metaPointable;
+
+ public FlushColumnTupleWithMetaWriter(FlushColumnMetadata columnMetadata, int pageSize, int maxNumberOfTuples,
+ float tolerance) {
+ super(columnMetadata, pageSize, maxNumberOfTuples, tolerance);
+ metaColumnTransformer = new ColumnTransformer(columnMetadata, columnMetadata.getMetaRoot());
+ metaPointable = new TypedRecordLazyVisitablePointable(columnMetadata.getMetaType());
+ }
+
+ @Override
+ protected void writeMeta(LSMBTreeTupleReference btreeTuple) throws HyracksDataException {
+ if (btreeTuple.isAntimatter()) {
+ return;
+ }
+
+ int metaFieldId = columnMetadata.getMetaRecordFieldIndex();
+ metaPointable.set(btreeTuple.getFieldData(metaFieldId), btreeTuple.getFieldStart(metaFieldId),
+ btreeTuple.getFieldLength(metaFieldId));
+ //In case the primary key is not in the meta part, we take the maximum
+ primaryKeysEstimatedSize = Math.max(metaColumnTransformer.transform(metaPointable), primaryKeysEstimatedSize);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
new file mode 100644
index 0000000..1af043f
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.flush;
+
+import java.nio.ByteBuffer;
+
+import org.apache.asterix.column.values.writer.ColumnBatchWriter;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.om.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.lazy.TypedRecordLazyVisitablePointable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
+
+public class FlushColumnTupleWriter extends AbstractColumnTupleWriter {
+ protected final FlushColumnMetadata columnMetadata;
+ protected final BatchFinalizerVisitor finalizer;
+ protected final ColumnBatchWriter writer;
+
+ private final ColumnTransformer transformer;
+ private final RecordLazyVisitablePointable pointable;
+ private final int maxNumberOfTuples;
+
+ protected int primaryKeysEstimatedSize;
+
+ public FlushColumnTupleWriter(FlushColumnMetadata columnMetadata, int pageSize, int maxNumberOfTuples,
+ float tolerance) {
+ this.columnMetadata = columnMetadata;
+ transformer = new ColumnTransformer(columnMetadata, columnMetadata.getRoot());
+ finalizer = new BatchFinalizerVisitor(columnMetadata);
+ writer = new ColumnBatchWriter(columnMetadata.getMultiPageOpRef(), pageSize, tolerance);
+ this.maxNumberOfTuples = maxNumberOfTuples;
+ pointable = new TypedRecordLazyVisitablePointable(columnMetadata.getDatasetType());
+ }
+
+ @Override
+ public final void init(IColumnWriteMultiPageOp multiPageOp) throws HyracksDataException {
+ columnMetadata.init(multiPageOp);
+ }
+
+ @Override
+ public final int getNumberOfColumns() {
+ return columnMetadata.getNumberOfColumns();
+ }
+
+ @Override
+ public final int bytesRequired(ITupleReference tuple) {
+ int primaryKeysSize = 0;
+ for (int i = 0; i < columnMetadata.getNumberOfPrimaryKeys(); i++) {
+ primaryKeysSize += tuple.getFieldLength(i);
+ }
+
+ //Mostly it is an overestimated size
+ return primaryKeysSize;
+ }
+
+ @Override
+ public final int getOccupiedSpace() {
+ int numberOfColumns = getNumberOfColumns();
+ int filterSize = numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE;
+ return primaryKeysEstimatedSize + filterSize;
+ }
+
+ @Override
+ public final int getMaxNumberOfTuples() {
+ return maxNumberOfTuples;
+ }
+
+ @Override
+ public final void close() {
+ columnMetadata.close();
+ }
+
+ @Override
+ public void writeTuple(ITupleReference tuple) throws HyracksDataException {
+ //This from an in-memory component, hence the cast
+ LSMBTreeTupleReference btreeTuple = (LSMBTreeTupleReference) tuple;
+ if (btreeTuple.isAntimatter()) {
+ //Write only the primary keys of an anti-matter tuple
+ primaryKeysEstimatedSize = transformer.writeAntiMatter(btreeTuple);
+ return;
+ }
+ writeRecord(tuple);
+ writeMeta(btreeTuple);
+ }
+
+ @Override
+ public final int flush(ByteBuffer pageZero) throws HyracksDataException {
+ writer.setPageZeroBuffer(pageZero, getNumberOfColumns(), columnMetadata.getNumberOfPrimaryKeys());
+ return finalizer.finalizeBatch(writer, columnMetadata);
+ }
+
+ protected void writeRecord(ITupleReference tuple) throws HyracksDataException {
+ int recordFieldId = columnMetadata.getRecordFieldIndex();
+ pointable.set(tuple.getFieldData(recordFieldId), tuple.getFieldStart(recordFieldId),
+ tuple.getFieldLength(recordFieldId));
+ primaryKeysEstimatedSize = transformer.transform(pointable);
+ }
+
+ protected void writeMeta(LSMBTreeTupleReference btreeTuple) throws HyracksDataException {
+ //NoOp
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/load/LoadColumnTupleReaderWriterFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/load/LoadColumnTupleReaderWriterFactory.java
new file mode 100644
index 0000000..0c1990f
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/load/LoadColumnTupleReaderWriterFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.load;
+
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnTupleReaderWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+
+public class LoadColumnTupleReaderWriterFactory extends FlushColumnTupleReaderWriterFactory {
+ private static final long serialVersionUID = -7583574057314353873L;
+
+ public LoadColumnTupleReaderWriterFactory(int pageSize, int maxNumberOfTuples, float tolerance) {
+ super(pageSize, maxNumberOfTuples, tolerance);
+ }
+
+ @Override
+ public AbstractColumnTupleWriter createColumnWriter(IColumnMetadata columnMetadata) {
+ return new LoadColumnTupleWriter((FlushColumnMetadata) columnMetadata, pageSize, maxNumberOfTuples, tolerance);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/load/LoadColumnTupleWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/load/LoadColumnTupleWriter.java
new file mode 100644
index 0000000..e4604da
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/load/LoadColumnTupleWriter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.load;
+
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnTupleWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class LoadColumnTupleWriter extends FlushColumnTupleWriter {
+ public LoadColumnTupleWriter(FlushColumnMetadata columnMetadata, int pageSize, int maxNumberOfTuples,
+ float tolerance) {
+ super(columnMetadata, pageSize, maxNumberOfTuples, tolerance);
+ }
+
+ @Override
+ public void writeTuple(ITupleReference tuple) throws HyracksDataException {
+ writeRecord(tuple);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/IEndOfPageCallBack.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/IEndOfPageCallBack.java
new file mode 100644
index 0000000..93df021
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/IEndOfPageCallBack.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.merge;
+
+import org.apache.asterix.column.tuple.MergeColumnTupleReference;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeRangeSearchCursor;
+
+/**
+ * An interface to signal {@link MergeColumnTupleWriter} that a component's page has reached the end.
+ */
+@FunctionalInterface
+public interface IEndOfPageCallBack {
+ /**
+ * Call {@link MergeColumnTupleWriter} to finish the current "vertical" merging batch.
+ * The caller of this method is {@link MergeColumnTupleReference#lastTupleReached()}
+ *
+ * @see ColumnBTreeRangeSearchCursor#doHasNext()
+ */
+ void callEnd(MergeColumnTupleReference columnTuple) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnReadMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnReadMetadata.java
new file mode 100644
index 0000000..11f3059
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnReadMetadata.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.merge;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+
+/**
+ * Merge column read metadata belongs to read an {@link ILSMDiskComponent}
+ * This only for reading an existing on-disk component for a merge operation. The schema here is immutable and cannot
+ * be changed.
+ */
+public final class MergeColumnReadMetadata extends AbstractColumnImmutableReadMetadata {
+ private final IColumnValuesReader[] columnReaders;
+
+ private MergeColumnReadMetadata(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ IColumnValuesReader[] columnReaders, IValueReference serializedMetadata) {
+ super(datasetType, metaType, numberOfPrimaryKeys, serializedMetadata, columnReaders.length);
+ this.columnReaders = columnReaders;
+ }
+
+ /**
+ * create ColumnMergeReadMetadata from columnMetadata
+ *
+ * @param serializedMetadata columnMetadata
+ * @return {@link MergeColumnReadMetadata}
+ * @see FlushColumnMetadata#serializeColumnsMetadata() for more information about serialization order
+ */
+ public static MergeColumnReadMetadata create(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ IColumnValuesReaderFactory readerFactory, IValueReference serializedMetadata) throws IOException {
+ byte[] bytes = serializedMetadata.getByteArray();
+ int offset = serializedMetadata.getStartOffset();
+ int length = serializedMetadata.getLength();
+
+ int pathInfoStart = offset + IntegerPointable.getInteger(bytes, offset + PATH_INFO_POINTER);
+ DataInput input = new DataInputStream(new ByteArrayInputStream(bytes, pathInfoStart, length));
+ int numberOfColumns = input.readInt();
+ IColumnValuesReader[] columnReaders = new IColumnValuesReader[numberOfColumns];
+ for (int i = 0; i < numberOfColumns; i++) {
+ IColumnValuesReader columnReader = readerFactory.createValueReader(input);
+ //The order at which the path info was written is not ordered by the column index
+ columnReaders[columnReader.getColumnIndex()] = columnReader;
+ }
+
+ return new MergeColumnReadMetadata(datasetType, metaType, numberOfPrimaryKeys, columnReaders,
+ serializedMetadata);
+ }
+
+ public IColumnValuesReader[] getColumnReaders() {
+ return columnReaders;
+ }
+
+ @Override
+ public int getColumnIndex(int ordinal) {
+ return ordinal;
+ }
+
+ @Override
+ public int getNumberOfProjectedColumns() {
+ return columnReaders.length;
+ }
+
+ @Override
+ public AbstractColumnTupleReader createTupleReader() {
+ return new MergeColumnTupleReader(this);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleProjector.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleProjector.java
new file mode 100644
index 0000000..f03506e
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleProjector.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.merge;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+
+public class MergeColumnTupleProjector implements IColumnTupleProjector {
+ private final ARecordType datasetType;
+ private final ARecordType metaType;
+ private final int numberOfPrimaryKeys;
+ private final IColumnValuesReaderFactory readerFactory;
+
+ public MergeColumnTupleProjector(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ IColumnValuesReaderFactory readerFactory) {
+ this.datasetType = datasetType;
+ this.metaType = metaType;
+ this.numberOfPrimaryKeys = numberOfPrimaryKeys;
+ this.readerFactory = readerFactory;
+ }
+
+ @Override
+ public IColumnProjectionInfo createProjectionInfo(IValueReference columnMetadata) throws HyracksDataException {
+ try {
+ return MergeColumnReadMetadata.create(datasetType, metaType, numberOfPrimaryKeys, readerFactory,
+ columnMetadata);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public ITupleReference project(ITupleReference tuple, DataOutput dos, ArrayTupleBuilder tb) throws IOException {
+ throw new IllegalAccessError(getClass().getName());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReader.java
new file mode 100644
index 0000000..4114f10
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReader.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.merge;
+
+import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
+import org.apache.asterix.column.tuple.MergeColumnTupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+
+public class MergeColumnTupleReader extends AbstractColumnTupleReader {
+ private final MergeColumnReadMetadata columnMetadata;
+
+ public MergeColumnTupleReader(AbstractColumnImmutableReadMetadata columnMetadata) {
+ this.columnMetadata = (MergeColumnReadMetadata) columnMetadata;
+ }
+
+ @Override
+ public IColumnTupleIterator createTupleIterator(ColumnBTreeReadLeafFrame frame, int componentIndex,
+ IColumnReadMultiPageOp multiPageOp) {
+ return new MergeColumnTupleReference(componentIndex, frame, columnMetadata, multiPageOp);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReaderWriterFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReaderWriterFactory.java
new file mode 100644
index 0000000..1ac94fe
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReaderWriterFactory.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.merge;
+
+import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReaderWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+
+public class MergeColumnTupleReaderWriterFactory extends AbstractColumnTupleReaderWriterFactory {
+ private static final long serialVersionUID = -2131401304338796428L;
+
+ public MergeColumnTupleReaderWriterFactory(int pageSize, int maxNumberOfTuples, float tolerance) {
+ super(pageSize, maxNumberOfTuples, tolerance);
+ }
+
+ @Override
+ public AbstractColumnTupleWriter createColumnWriter(IColumnMetadata columnMetadata) {
+ MergeColumnWriteMetadata mergeWriteMetadata = (MergeColumnWriteMetadata) columnMetadata;
+ return new MergeColumnTupleWriter(mergeWriteMetadata, pageSize, maxNumberOfTuples, tolerance);
+ }
+
+ @Override
+ public AbstractColumnTupleReader createColumnReader(IColumnProjectionInfo columnProjectionInfo) {
+ return ((AbstractColumnImmutableReadMetadata) columnProjectionInfo).createTupleReader();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java
new file mode 100644
index 0000000..fbda6d0
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.merge;
+
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import org.apache.asterix.column.tuple.MergeColumnTupleReference;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.writer.ColumnBatchWriter;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+public class MergeColumnTupleWriter extends AbstractColumnTupleWriter {
+ private final MergeColumnWriteMetadata columnMetadata;
+ private final MergeColumnTupleReference[] componentsTuples;
+ private final RunLengthIntArray writtenComponents;
+
+ private final IColumnValuesWriter[] primaryKeyWriters;
+ private final PriorityQueue<IColumnValuesWriter> orderedColumns;
+ private final ColumnBatchWriter writer;
+ private final int maxNumberOfTuples;
+ private int primaryKeysEstimatedSize;
+
+ public MergeColumnTupleWriter(MergeColumnWriteMetadata columnMetadata, int pageSize, int maxNumberOfTuples,
+ float tolerance) {
+ this.columnMetadata = columnMetadata;
+ List<IColumnTupleIterator> componentsTuplesList = columnMetadata.getComponentsTuples();
+ this.componentsTuples = new MergeColumnTupleReference[componentsTuplesList.size()];
+ for (int i = 0; i < componentsTuplesList.size(); i++) {
+ MergeColumnTupleReference mergeTuple = (MergeColumnTupleReference) componentsTuplesList.get(i);
+ this.componentsTuples[i] = mergeTuple;
+ mergeTuple.registerEndOfPageCallBack(this::writeAllColumns);
+ }
+ this.writtenComponents = new RunLengthIntArray();
+ this.maxNumberOfTuples = maxNumberOfTuples;
+ writer = new ColumnBatchWriter(columnMetadata.getMultiPageOpRef(), pageSize, tolerance);
+ writtenComponents.reset();
+
+ primaryKeyWriters = new IColumnValuesWriter[columnMetadata.getNumberOfPrimaryKeys()];
+ for (int i = 0; i < primaryKeyWriters.length; i++) {
+ primaryKeyWriters[i] = columnMetadata.getWriter(i);
+ }
+ orderedColumns = new PriorityQueue<>(Comparator.comparingInt(x -> -x.getEstimatedSize()));
+ }
+
+ @Override
+ public int bytesRequired(ITupleReference tuple) {
+ int primaryKeysSize = 0;
+ for (int i = 0; i < columnMetadata.getNumberOfPrimaryKeys(); i++) {
+ primaryKeysSize += tuple.getFieldLength(i);
+ }
+
+ return primaryKeysSize;
+ }
+
+ @Override
+ public void init(IColumnWriteMultiPageOp multiPageOp) throws HyracksDataException {
+ columnMetadata.init(multiPageOp);
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return columnMetadata.getNumberOfColumns();
+ }
+
+ @Override
+ public int getMaxNumberOfTuples() {
+ return maxNumberOfTuples;
+ }
+
+ @Override
+ public int getOccupiedSpace() {
+ int numberOfColumns = getNumberOfColumns();
+ int filterSize = numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE;
+ return primaryKeysEstimatedSize + filterSize;
+ }
+
+ @Override
+ public void writeTuple(ITupleReference tuple) throws HyracksDataException {
+ MergeColumnTupleReference columnTuple = (MergeColumnTupleReference) tuple;
+ int componentIndex = columnTuple.getComponentIndex();
+ int skipCount = columnTuple.getAndResetSkipCount();
+ if (skipCount > 0) {
+ writtenComponents.add(-componentIndex, skipCount);
+ }
+ if (columnTuple.isAntimatter()) {
+ writtenComponents.add(-componentIndex);
+ } else {
+ writtenComponents.add(componentIndex);
+ }
+ writePrimaryKeys(columnTuple);
+ }
+
+ private void writePrimaryKeys(MergeColumnTupleReference columnTuple) throws HyracksDataException {
+ int primaryKeySize = 0;
+ for (int i = 0; i < columnMetadata.getNumberOfPrimaryKeys(); i++) {
+ IColumnValuesReader columnReader = columnTuple.getReader(i);
+ IColumnValuesWriter columnWriter = primaryKeyWriters[i];
+ columnReader.write(columnWriter, false);
+ primaryKeySize += columnWriter.getEstimatedSize();
+ }
+ primaryKeysEstimatedSize = primaryKeySize;
+ }
+
+ private void writeNonKeyColumns() throws HyracksDataException {
+ for (int i = 0; i < writtenComponents.getNumberOfBlocks(); i++) {
+ int componentIndex = writtenComponents.getBlockValue(i);
+ if (componentIndex < 0) {
+ //Skip writing values of deleted tuples
+ componentIndex = -componentIndex;
+ skipReaders(componentIndex, writtenComponents.getBlockSize(i));
+ continue;
+ }
+ MergeColumnTupleReference componentTuple = componentsTuples[componentIndex];
+ int count = writtenComponents.getBlockSize(i);
+ for (int j = columnMetadata.getNumberOfPrimaryKeys(); j < columnMetadata.getNumberOfColumns(); j++) {
+ IColumnValuesReader columnReader = componentTuple.getReader(j);
+ IColumnValuesWriter columnWriter = columnMetadata.getWriter(j);
+ columnReader.write(columnWriter, count);
+ }
+ }
+ }
+
+ private void skipReaders(int componentIndex, int count) throws HyracksDataException {
+ MergeColumnTupleReference componentTuple = componentsTuples[componentIndex];
+ for (int j = columnMetadata.getNumberOfPrimaryKeys(); j < columnMetadata.getNumberOfColumns(); j++) {
+ IColumnValuesReader columnReader = componentTuple.getReader(j);
+ columnReader.skip(count);
+ }
+ }
+
+ @Override
+ public int flush(ByteBuffer pageZero) throws HyracksDataException {
+ int numberOfColumns = columnMetadata.getNumberOfColumns();
+ int numberOfPrimaryKeys = columnMetadata.getNumberOfPrimaryKeys();
+ if (writtenComponents.getSize() > 0) {
+ writeNonKeyColumns();
+ writtenComponents.reset();
+ }
+ for (int i = numberOfPrimaryKeys; i < numberOfColumns; i++) {
+ orderedColumns.add(columnMetadata.getWriter(i));
+ }
+ writer.setPageZeroBuffer(pageZero, numberOfColumns, numberOfPrimaryKeys);
+ int allocatedSpace = writer.writePrimaryKeyColumns(primaryKeyWriters);
+ allocatedSpace += writer.writeColumns(orderedColumns);
+ return allocatedSpace;
+ }
+
+ @Override
+ public void close() {
+ columnMetadata.close();
+ }
+
+ private void writeAllColumns(MergeColumnTupleReference columnTuple) throws HyracksDataException {
+ /*
+ * The last tuple from one of the components was reached. Since we are going to the next leaf, we will not be
+ * able to access the readers of this component's leaf after this tuple. So, we are going to write
+ * the values of all columns as recorded in writtenComponents
+ */
+ int skipCount = columnTuple.getAndResetSkipCount();
+ if (skipCount > 0) {
+ writtenComponents.add(-columnTuple.getComponentIndex(), skipCount);
+ }
+ writeNonKeyColumns();
+ writtenComponents.reset();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnWriteMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnWriteMetadata.java
new file mode 100644
index 0000000..b0d1a01
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnWriteMetadata.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.lsm.merge;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.column.metadata.AbstractColumnImmutableMetadata;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.IColumnValuesWriterFactory;
+import org.apache.asterix.column.values.writer.ColumnValuesWriterFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+
+/**
+ * Merge column write metadata belongs to write a new merge {@link ILSMDiskComponent}
+ * This is for writing a new on-disk component by merging two or more on disk components. The final schema for this
+ * component will the most recent schema, which belongs to the newest merged component. The schema here is immutable
+ * and cannot be changed.
+ */
+public final class MergeColumnWriteMetadata extends AbstractColumnImmutableMetadata {
+ private final Mutable<IColumnWriteMultiPageOp> multiPageOpRef;
+ private final List<IColumnValuesWriter> columnWriters;
+ private final List<IColumnTupleIterator> componentsTuples;
+
+ /**
+ * For LSM Merge
+ */
+ private MergeColumnWriteMetadata(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef, List<IColumnValuesWriter> columnWriters,
+ IValueReference serializedMetadata, List<IColumnTupleIterator> componentsTuples) {
+ super(datasetType, metaType, numberOfPrimaryKeys, serializedMetadata, columnWriters.size());
+ this.multiPageOpRef = multiPageOpRef;
+ this.columnWriters = columnWriters;
+ this.componentsTuples = componentsTuples;
+ }
+
+ /**
+ * Set {@link IColumnWriteMultiPageOp} for {@link IColumnValuesWriter}
+ *
+ * @param multiPageOp multi-buffer allocator
+ */
+ public void init(IColumnWriteMultiPageOp multiPageOp) throws HyracksDataException {
+ multiPageOpRef.setValue(multiPageOp);
+
+ //Reset writer for the first write
+ for (int i = 0; i < columnWriters.size(); i++) {
+ columnWriters.get(i).reset();
+ }
+ }
+
+ public Mutable<IColumnWriteMultiPageOp> getMultiPageOpRef() {
+ return multiPageOpRef;
+ }
+
+ public IColumnValuesWriter getWriter(int columnIndex) {
+ return columnWriters.get(columnIndex);
+ }
+
+ public void close() {
+ multiPageOpRef.setValue(null);
+ for (int i = 0; i < columnWriters.size(); i++) {
+ columnWriters.get(i).close();
+ }
+ }
+
+ public static MergeColumnWriteMetadata create(ARecordType datasetType, ARecordType metaType,
+ int numberOfPrimaryKeys, Mutable<IColumnWriteMultiPageOp> multiPageOpRef,
+ IValueReference serializedMetadata, List<IColumnTupleIterator> componentsTuples) throws IOException {
+ byte[] bytes = serializedMetadata.getByteArray();
+ int offset = serializedMetadata.getStartOffset();
+ int length = serializedMetadata.getLength();
+
+ int writersOffset = offset + IntegerPointable.getInteger(bytes, offset + WRITERS_POINTER);
+ DataInput input = new DataInputStream(new ByteArrayInputStream(bytes, writersOffset, length));
+
+ IColumnValuesWriterFactory writerFactory = new ColumnValuesWriterFactory(multiPageOpRef);
+ List<IColumnValuesWriter> writers = new ArrayList<>();
+ FlushColumnMetadata.deserializeWriters(input, writers, writerFactory);
+
+ return new MergeColumnWriteMetadata(datasetType, metaType, numberOfPrimaryKeys, multiPageOpRef, writers,
+ serializedMetadata, componentsTuples);
+ }
+
+ public List<IColumnTupleIterator> getComponentsTuples() {
+ return componentsTuples;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
new file mode 100644
index 0000000..c329d67
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.query;
+
+import java.util.List;
+
+import org.apache.asterix.column.assembler.AbstractPrimitiveValueAssembler;
+import org.apache.asterix.column.assembler.AssemblerBuilderVisitor;
+import org.apache.asterix.column.assembler.ObjectValueAssembler;
+import org.apache.asterix.column.assembler.value.IValueGetterFactory;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public final class ColumnAssembler {
+ private final List<AbstractPrimitiveValueAssembler> assemblers;
+ private final ObjectValueAssembler rootAssembler;
+ private int numberOfTuples;
+ private int tupleIndex;
+
+ public ColumnAssembler(AbstractSchemaNode node, ARecordType declaredType, QueryColumnMetadata columnMetadata,
+ IColumnValuesReaderFactory readerFactory, IValueGetterFactory valueGetterFactory)
+ throws HyracksDataException {
+ AssemblerBuilderVisitor builderVisitor =
+ new AssemblerBuilderVisitor(columnMetadata, readerFactory, valueGetterFactory);
+ assemblers = builderVisitor.createValueAssemblers(node, declaredType);
+ rootAssembler = (ObjectValueAssembler) builderVisitor.getRootAssembler();
+ }
+
+ public void reset(int numberOfTuples) {
+ this.numberOfTuples = numberOfTuples;
+ tupleIndex = 0;
+ }
+
+ public void resetColumn(AbstractBytesInputStream stream, int startIndex, int ordinal) throws HyracksDataException {
+ assemblers.get(ordinal).reset(stream, startIndex, numberOfTuples);
+ }
+
+ public int getColumnIndex(int ordinal) {
+ return assemblers.get(ordinal).getColumnIndex();
+ }
+
+ public boolean hasNext() {
+ return tupleIndex < numberOfTuples;
+ }
+
+ public IValueReference nextValue() throws HyracksDataException {
+ rootAssembler.start();
+ if (tupleIndex == numberOfTuples) {
+ rootAssembler.end();
+ //return empty record
+ return rootAssembler.getValue();
+ }
+
+ int index = 0;
+ while (index < assemblers.size()) {
+ AbstractPrimitiveValueAssembler assembler = assemblers.get(index);
+ int groupIndex = assembler.next();
+ if (groupIndex != AbstractPrimitiveValueAssembler.NEXT_ASSEMBLER) {
+ index = groupIndex;
+ } else {
+ index++;
+ }
+ }
+
+ tupleIndex++;
+ rootAssembler.end();
+ return rootAssembler.getValue();
+ }
+
+ public int getNumberOfColumns() {
+ return assemblers.size();
+ }
+
+ public void skip(int count) throws HyracksDataException {
+ for (int i = 0; i < assemblers.size(); i++) {
+ assemblers.get(i).skip(count);
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
new file mode 100644
index 0000000..6c51240
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.query;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.column.assembler.value.IValueGetterFactory;
+import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
+import org.apache.asterix.column.metadata.FieldNamesDictionary;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.evaluator.FalseColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.evaluator.TrueColumnFilterEvaluator;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Query column metadata which is used to resolve the requested values in a query
+ */
+public class QueryColumnMetadata extends AbstractColumnImmutableReadMetadata {
+ private static final Logger LOGGER = LogManager.getLogger();
+ private final FieldNamesDictionary fieldNamesDictionary;
+ private final IColumnValuesReader[] primaryKeyReaders;
+ private final IColumnFilterEvaluator filterEvaluator;
+ private final List<IColumnFilterValueAccessor> filterValueAccessors;
+
+ protected final ColumnAssembler assembler;
+
+ protected QueryColumnMetadata(ARecordType datasetType, ARecordType metaType,
+ IColumnValuesReader[] primaryKeyReaders, IValueReference serializedMetadata,
+ FieldNamesDictionary fieldNamesDictionary, ObjectSchemaNode root, IColumnValuesReaderFactory readerFactory,
+ IValueGetterFactory valueGetterFactory, IColumnFilterEvaluator filterEvaluator,
+ List<IColumnFilterValueAccessor> filterValueAccessors) throws HyracksDataException {
+ super(datasetType, metaType, primaryKeyReaders.length, serializedMetadata, -1);
+ this.fieldNamesDictionary = fieldNamesDictionary;
+ this.assembler = new ColumnAssembler(root, datasetType, this, readerFactory, valueGetterFactory);
+ this.primaryKeyReaders = primaryKeyReaders;
+ this.filterEvaluator = filterEvaluator;
+ this.filterValueAccessors = filterValueAccessors;
+ }
+
+ public final ColumnAssembler getAssembler() {
+ return assembler;
+ }
+
+ public final FieldNamesDictionary getFieldNamesDictionary() {
+ return fieldNamesDictionary;
+ }
+
+ public final IColumnValuesReader[] getPrimaryKeyReaders() {
+ return primaryKeyReaders;
+ }
+
+ public IColumnFilterEvaluator getFilterEvaluator() {
+ return filterEvaluator;
+ }
+
+ public List<IColumnFilterValueAccessor> getFilterValueAccessors() {
+ return filterValueAccessors;
+ }
+
+ /* *****************************************************
+ * Non-final methods
+ * *****************************************************
+ */
+
+ public boolean containsMeta() {
+ return false;
+ }
+
+ @Override
+ public int getColumnIndex(int ordinal) {
+ return assembler.getColumnIndex(ordinal);
+ }
+
+ @Override
+ public int getNumberOfProjectedColumns() {
+ return assembler.getNumberOfColumns();
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return assembler.getNumberOfColumns();
+ }
+
+ @Override
+ public AbstractColumnTupleReader createTupleReader() {
+ return new QueryColumnTupleReader(this);
+ }
+
+ /**
+ * Create {@link QueryColumnMetadata} that would be used to determine the requested values
+ *
+ * @param datasetType dataset declared type
+ * @param numberOfPrimaryKeys number of PKs
+ * @param serializedMetadata inferred metadata (schema)
+ * @param readerFactory column reader factory
+ * @param valueGetterFactory value serializer
+ * @param requestedType the requested schema
+ * @return query metadata
+ */
+ public static QueryColumnMetadata create(ARecordType datasetType, int numberOfPrimaryKeys,
+ IValueReference serializedMetadata, IColumnValuesReaderFactory readerFactory,
+ IValueGetterFactory valueGetterFactory, ARecordType requestedType,
+ Map<String, FunctionCallInformation> functionCallInfoMap,
+ IColumnFilterEvaluatorFactory filterEvaluatorFactory, IWarningCollector warningCollector)
+ throws IOException {
+ byte[] bytes = serializedMetadata.getByteArray();
+ int offset = serializedMetadata.getStartOffset();
+ int length = serializedMetadata.getLength();
+
+ int fieldNamesStart = offset + IntegerPointable.getInteger(bytes, offset + FIELD_NAMES_POINTER);
+ int metaRootStart = IntegerPointable.getInteger(bytes, offset + META_SCHEMA_POINTER);
+ int metaRootSize =
+ metaRootStart < 0 ? 0 : IntegerPointable.getInteger(bytes, offset + PATH_INFO_POINTER) - metaRootStart;
+ DataInput input = new DataInputStream(new ByteArrayInputStream(bytes, fieldNamesStart, length));
+
+ //FieldNames
+ FieldNamesDictionary fieldNamesDictionary = FieldNamesDictionary.deserialize(input);
+
+ //Schema
+ ObjectSchemaNode root = (ObjectSchemaNode) AbstractSchemaNode.deserialize(input, null);
+ //Skip metaRoot (if exists)
+ input.skipBytes(metaRootSize);
+
+ //Clip schema
+ SchemaClipperVisitor clipperVisitor =
+ new SchemaClipperVisitor(fieldNamesDictionary, functionCallInfoMap, warningCollector);
+ ObjectSchemaNode clippedRoot = clip(requestedType, root, clipperVisitor);
+
+ FilterAccessorProvider filterAccessorProvider = new FilterAccessorProvider(root, clipperVisitor);
+ IColumnFilterEvaluator filterEvaluator = filterEvaluatorFactory.create(filterAccessorProvider);
+ List<IColumnFilterValueAccessor> filterValueAccessors = filterAccessorProvider.getFilterAccessors();
+
+ IColumnValuesReader[] primaryKeyReaders = createPrimaryKeyReaders(input, readerFactory, numberOfPrimaryKeys);
+
+ if (LOGGER.isInfoEnabled() && filterEvaluator != TrueColumnFilterEvaluator.INSTANCE) {
+ String filterString = filterEvaluator == FalseColumnFilterEvaluator.INSTANCE ? "SKIP_ALL"
+ : filterEvaluatorFactory.toString();
+ LOGGER.info("Filter: {}", filterString);
+ }
+
+ return new QueryColumnMetadata(datasetType, null, primaryKeyReaders, serializedMetadata, fieldNamesDictionary,
+ clippedRoot, readerFactory, valueGetterFactory, filterEvaluator, filterValueAccessors);
+ }
+
+ protected static ObjectSchemaNode clip(ARecordType requestedType, ObjectSchemaNode root,
+ SchemaClipperVisitor clipperVisitor) {
+ ObjectSchemaNode clippedRoot;
+ if (requestedType.getTypeName().equals(DataProjectionFiltrationInfo.ALL_FIELDS_TYPE.getTypeName())) {
+ clippedRoot = root;
+ } else {
+ clippedRoot = (ObjectSchemaNode) requestedType.accept(clipperVisitor, root);
+ }
+ return clippedRoot;
+ }
+
+ protected static IColumnValuesReader[] createPrimaryKeyReaders(DataInput input,
+ IColumnValuesReaderFactory readerFactory, int numberOfPrimaryKeys) throws IOException {
+ //skip number of columns
+ input.readInt();
+
+ IColumnValuesReader[] primaryKeyReaders = new IColumnValuesReader[numberOfPrimaryKeys];
+ for (int i = 0; i < numberOfPrimaryKeys; i++) {
+ primaryKeyReaders[i] = readerFactory.createValueReader(input);
+ }
+ return primaryKeyReaders;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java
new file mode 100644
index 0000000..8865b13
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.query;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.column.assembler.value.ValueGetterFactory;
+import org.apache.asterix.column.tuple.AssembledTupleReference;
+import org.apache.asterix.column.tuple.QueryColumnTupleReference;
+import org.apache.asterix.column.values.reader.ColumnValueReaderFactory;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+
+class QueryColumnTupleProjector implements IColumnTupleProjector {
+ protected final ARecordType datasetType;
+ protected final ARecordType requestedType;
+ protected final int numberOfPrimaryKeys;
+ protected final Map<String, FunctionCallInformation> functionCallInfoMap;
+ protected final IWarningCollector warningCollector;
+ protected final IColumnFilterEvaluatorFactory filterEvaluator;
+ private final AssembledTupleReference assembledTupleReference;
+
+ QueryColumnTupleProjector(ARecordType datasetType, int numberOfPrimaryKeys, ARecordType requestedType,
+ Map<String, FunctionCallInformation> functionCallInfoMap, IColumnFilterEvaluatorFactory filterEvaluator,
+ IWarningCollector warningCollector) {
+ this.datasetType = datasetType;
+ this.numberOfPrimaryKeys = numberOfPrimaryKeys;
+ this.requestedType = requestedType;
+ this.functionCallInfoMap = functionCallInfoMap;
+ this.filterEvaluator = filterEvaluator;
+ this.warningCollector = warningCollector;
+ assembledTupleReference = new AssembledTupleReference(getNumberOfTupleFields());
+ }
+
+ @Override
+ public IColumnProjectionInfo createProjectionInfo(IValueReference serializedMetadata) throws HyracksDataException {
+ try {
+ return QueryColumnMetadata.create(datasetType, numberOfPrimaryKeys, serializedMetadata,
+ new ColumnValueReaderFactory(), ValueGetterFactory.INSTANCE, requestedType, functionCallInfoMap,
+ filterEvaluator, warningCollector);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public final ITupleReference project(ITupleReference tuple, DataOutput dos, ArrayTupleBuilder tb)
+ throws IOException {
+ for (int i = 0; i < numberOfPrimaryKeys; i++) {
+ dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ if (isColumnar(tuple)) {
+ IValueReference assembledRecord = getAssembledValue(tuple);
+ dos.write(assembledRecord.getByteArray(), assembledRecord.getStartOffset(), assembledRecord.getLength());
+ } else {
+ dos.write(tuple.getFieldData(numberOfPrimaryKeys), tuple.getFieldStart(numberOfPrimaryKeys),
+ tuple.getFieldLength(numberOfPrimaryKeys));
+ }
+ tb.addFieldEndOffset();
+ //Write meta (if any)
+ writeMeta(tuple, dos, tb);
+
+ return assembledTupleReference.reset(tb);
+ }
+
+ protected boolean isColumnar(ITupleReference tuple) {
+ return tuple instanceof QueryColumnTupleReference;
+ }
+
+ protected IValueReference getAssembledValue(ITupleReference tuple) throws HyracksDataException {
+ QueryColumnTupleReference columnTuple = (QueryColumnTupleReference) tuple;
+ return columnTuple.getAssembledValue();
+ }
+
+ protected void writeMeta(ITupleReference tuple, DataOutput dos, ArrayTupleBuilder tb) throws IOException {
+ //NoOp
+ }
+
+ protected int getNumberOfTupleFields() {
+ return numberOfPrimaryKeys + 1;
+ }
+
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjectorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjectorFactory.java
new file mode 100644
index 0000000..7daa877
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjectorFactory.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.query;
+
+import java.util.Map;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.storage.common.projection.ITupleProjector;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
+
+public class QueryColumnTupleProjectorFactory implements ITupleProjectorFactory {
+ private static final long serialVersionUID = 2130283796584264219L;
+ private final ARecordType datasetType;
+ private final ARecordType metaType;
+ private final int numberOfPrimaryKeys;
+ private final ARecordType requestedType;
+ private final ARecordType requestedMetaType;
+ private final Map<String, FunctionCallInformation> functionCallInfo;
+ private final Map<String, FunctionCallInformation> metaFunctionCallInfo;
+ private final IColumnFilterEvaluatorFactory filterEvaluator;
+
+ public QueryColumnTupleProjectorFactory(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ ARecordType requestedType, Map<String, FunctionCallInformation> functionCallInfo,
+ ARecordType requestedMetaType, Map<String, FunctionCallInformation> metaFunctionCallInfo,
+ IColumnFilterEvaluatorFactory filterEvaluator) {
+ this.datasetType = datasetType;
+ this.metaType = metaType;
+ this.numberOfPrimaryKeys = numberOfPrimaryKeys;
+ this.requestedType = requestedType;
+ this.functionCallInfo = functionCallInfo;
+ this.requestedMetaType = requestedMetaType;
+ this.metaFunctionCallInfo = metaFunctionCallInfo;
+ this.filterEvaluator = filterEvaluator;
+ }
+
+ @Override
+ public ITupleProjector createTupleProjector(IHyracksTaskContext context) throws HyracksDataException {
+ IWarningCollector warningCollector = context.getWarningCollector();
+ if (requestedMetaType == null
+ || DataProjectionFiltrationInfo.EMPTY_TYPE.getTypeName().equals(requestedMetaType.getTypeName())) {
+ /*
+ * Either the dataset does not contain meta record or none of the meta columns were requested. Thus,
+ * ignore reading the meta columns (if exist)
+ */
+ return new QueryColumnTupleProjector(datasetType, numberOfPrimaryKeys, requestedType, functionCallInfo,
+ filterEvaluator, warningCollector);
+ }
+ //The query requested some or all of the meta columns
+ return new QueryColumnWithMetaTupleProjector(datasetType, metaType, numberOfPrimaryKeys, requestedType,
+ functionCallInfo, requestedMetaType, metaFunctionCallInfo, filterEvaluator, warningCollector);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleReader.java
new file mode 100644
index 0000000..36e47ec
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleReader.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.query;
+
+import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
+import org.apache.asterix.column.tuple.QueryColumnTupleReference;
+import org.apache.asterix.column.tuple.QueryColumnWithMetaTupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+
+public class QueryColumnTupleReader extends AbstractColumnTupleReader {
+ private final QueryColumnMetadata columnMetadata;
+
+ public QueryColumnTupleReader(AbstractColumnImmutableReadMetadata columnMetadata) {
+ this.columnMetadata = (QueryColumnMetadata) columnMetadata;
+ }
+
+ @Override
+ public IColumnTupleIterator createTupleIterator(ColumnBTreeReadLeafFrame frame, int index,
+ IColumnReadMultiPageOp multiPageOp) {
+ if (columnMetadata.containsMeta()) {
+ return new QueryColumnWithMetaTupleReference(index, frame, columnMetadata, multiPageOp);
+ }
+ return new QueryColumnTupleReference(index, frame, columnMetadata, multiPageOp);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
new file mode 100644
index 0000000..6f0c974
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.query;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.column.assembler.value.IValueGetterFactory;
+import org.apache.asterix.column.metadata.FieldNamesDictionary;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+
+/**
+ * Query column metadata (with metaRecord)
+ */
+public final class QueryColumnWithMetaMetadata extends QueryColumnMetadata {
+ private final ColumnAssembler metaAssembler;
+
+ private QueryColumnWithMetaMetadata(ARecordType datasetType, ARecordType metaType,
+ IColumnValuesReader[] primaryKeyReaders, IValueReference serializedMetadata,
+ FieldNamesDictionary fieldNamesDictionary, ObjectSchemaNode root, ObjectSchemaNode metaRoot,
+ IColumnValuesReaderFactory readerFactory, IValueGetterFactory valueGetterFactory,
+ IColumnFilterEvaluator filterEvaluator, List<IColumnFilterValueAccessor> filterValueAccessors)
+ throws HyracksDataException {
+ super(datasetType, metaType, primaryKeyReaders, serializedMetadata, fieldNamesDictionary, root, readerFactory,
+ valueGetterFactory, filterEvaluator, filterValueAccessors);
+ metaAssembler = new ColumnAssembler(metaRoot, metaType, this, readerFactory, valueGetterFactory);
+ }
+
+ @Override
+ public boolean containsMeta() {
+ return true;
+ }
+
+ @Override
+ public int getColumnIndex(int ordinal) {
+ int metaColumnCount = metaAssembler.getNumberOfColumns();
+ if (ordinal >= metaColumnCount) {
+ return assembler.getColumnIndex(ordinal - metaColumnCount);
+ }
+ return metaAssembler.getColumnIndex(ordinal);
+ }
+
+ @Override
+ public int getNumberOfProjectedColumns() {
+ return metaAssembler.getNumberOfColumns() + assembler.getNumberOfColumns();
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return getNumberOfProjectedColumns();
+ }
+
+ @Override
+ public AbstractColumnTupleReader createTupleReader() {
+ return new QueryColumnTupleReader(this);
+ }
+
+ public ColumnAssembler getMetaAssembler() {
+ return metaAssembler;
+ }
+
+ /**
+ * Create {@link QueryColumnWithMetaMetadata} that would be used to determine the requested values
+ *
+ * @param datasetType dataset declared type
+ * @param metaType meta declared type
+ * @param numberOfPrimaryKeys number of PKs
+ * @param serializedMetadata inferred metadata (schema)
+ * @param readerFactory column reader factory
+ * @param valueGetterFactory value serializer
+ * @param requestedType the requested schema
+ * @return query metadata
+ */
+ public static QueryColumnWithMetaMetadata create(ARecordType datasetType, ARecordType metaType,
+ int numberOfPrimaryKeys, IValueReference serializedMetadata, IColumnValuesReaderFactory readerFactory,
+ IValueGetterFactory valueGetterFactory, ARecordType requestedType,
+ Map<String, FunctionCallInformation> functionCallInfo, ARecordType metaRequestedType,
+ Map<String, FunctionCallInformation> metaFunctionCallInfo,
+ IColumnFilterEvaluatorFactory filterEvaluatorFactory, IWarningCollector warningCollector)
+ throws IOException {
+ byte[] bytes = serializedMetadata.getByteArray();
+ int offset = serializedMetadata.getStartOffset();
+ int length = serializedMetadata.getLength();
+
+ int fieldNamesStart = offset + IntegerPointable.getInteger(bytes, offset + FIELD_NAMES_POINTER);
+ DataInput input = new DataInputStream(new ByteArrayInputStream(bytes, fieldNamesStart, length));
+
+ //FieldNames
+ FieldNamesDictionary fieldNamesDictionary = FieldNamesDictionary.deserialize(input);
+
+ //Schema
+ ObjectSchemaNode root = (ObjectSchemaNode) AbstractSchemaNode.deserialize(input, null);
+ ObjectSchemaNode metaRoot = (ObjectSchemaNode) AbstractSchemaNode.deserialize(input, null);
+
+ //Clip dataset schema
+ SchemaClipperVisitor clipperVisitor =
+ new SchemaClipperVisitor(fieldNamesDictionary, functionCallInfo, warningCollector);
+ ObjectSchemaNode clippedRoot = clip(requestedType, root, clipperVisitor);
+
+ //Clip meta schema
+ SchemaClipperVisitor metaClipperVisitor =
+ new SchemaClipperVisitor(fieldNamesDictionary, metaFunctionCallInfo, warningCollector);
+ ObjectSchemaNode metaClippedRoot = clip(metaRequestedType, metaRoot, metaClipperVisitor);
+
+ FilterAccessorProvider filterAccessorProvider = new FilterAccessorProvider(root, metaRoot, clipperVisitor);
+ IColumnFilterEvaluator filterEvaluator = filterEvaluatorFactory.create(filterAccessorProvider);
+ List<IColumnFilterValueAccessor> filterValueAccessors = filterAccessorProvider.getFilterAccessors();
+
+ IColumnValuesReader[] primaryKeyReaders = createPrimaryKeyReaders(input, readerFactory, numberOfPrimaryKeys);
+
+ return new QueryColumnWithMetaMetadata(datasetType, metaType, primaryKeyReaders, serializedMetadata,
+ fieldNamesDictionary, clippedRoot, metaClippedRoot, readerFactory, valueGetterFactory, filterEvaluator,
+ filterValueAccessors);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java
new file mode 100644
index 0000000..e8f9045
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.operation.query;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.column.assembler.value.ValueGetterFactory;
+import org.apache.asterix.column.tuple.QueryColumnWithMetaTupleReference;
+import org.apache.asterix.column.values.reader.ColumnValueReaderFactory;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+
+class QueryColumnWithMetaTupleProjector extends QueryColumnTupleProjector {
+ private final ARecordType metaType;
+ private final ARecordType requestedMetaType;
+ private final Map<String, FunctionCallInformation> metaFunctionCallInfoMap;
+
+ QueryColumnWithMetaTupleProjector(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ ARecordType requestedType, Map<String, FunctionCallInformation> functionCallInfoMap,
+ ARecordType requestedMetaType, Map<String, FunctionCallInformation> metaFunctionCallInfoMap,
+ IColumnFilterEvaluatorFactory filterEvaluator, IWarningCollector warningCollector) {
+ super(datasetType, numberOfPrimaryKeys, requestedType, functionCallInfoMap, filterEvaluator, warningCollector);
+ this.metaType = metaType;
+ this.requestedMetaType = requestedMetaType;
+ this.metaFunctionCallInfoMap = metaFunctionCallInfoMap;
+ }
+
+ @Override
+ public IColumnProjectionInfo createProjectionInfo(IValueReference serializedMetadata) throws HyracksDataException {
+ try {
+ return QueryColumnWithMetaMetadata.create(datasetType, metaType, numberOfPrimaryKeys, serializedMetadata,
+ new ColumnValueReaderFactory(), ValueGetterFactory.INSTANCE, requestedType, functionCallInfoMap,
+ requestedMetaType, metaFunctionCallInfoMap, filterEvaluator, warningCollector);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ protected boolean isColumnar(ITupleReference tuple) {
+ return tuple instanceof QueryColumnWithMetaTupleReference;
+ }
+
+ @Override
+ protected IValueReference getAssembledValue(ITupleReference tuple) throws HyracksDataException {
+ QueryColumnWithMetaTupleReference columnTuple = (QueryColumnWithMetaTupleReference) tuple;
+ return columnTuple.getAssembledValue();
+ }
+
+ @Override
+ protected void writeMeta(ITupleReference tuple, DataOutput dos, ArrayTupleBuilder tb) throws IOException {
+ if (tuple instanceof QueryColumnWithMetaTupleReference) {
+ QueryColumnWithMetaTupleReference columnTuple = (QueryColumnWithMetaTupleReference) tuple;
+ IValueReference assembledRecord = columnTuple.getMetaAssembledValue();
+ dos.write(assembledRecord.getByteArray(), assembledRecord.getStartOffset(), assembledRecord.getLength());
+ } else {
+ dos.write(tuple.getFieldData(numberOfPrimaryKeys + 1), tuple.getFieldStart(numberOfPrimaryKeys + 1),
+ tuple.getFieldLength(numberOfPrimaryKeys + 1));
+ }
+ tb.addFieldEndOffset();
+ }
+
+ @Override
+ protected int getNumberOfTupleFields() {
+ return super.getNumberOfTupleFields() + 1;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/AbstractAsterixColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/AbstractAsterixColumnTupleReference.java
new file mode 100644
index 0000000..df6b554
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/AbstractAsterixColumnTupleReference.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.tuple;
+
+import org.apache.asterix.column.assembler.value.IValueGetter;
+import org.apache.asterix.column.assembler.value.ValueGetterFactory;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream;
+import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples.AbstractColumnTupleReference;
+
+public abstract class AbstractAsterixColumnTupleReference extends AbstractColumnTupleReference {
+ private final IValueGetter[] primaryKeysValueGetters;
+ protected final ByteBufferInputStream[] primaryKeyStreams;
+ protected final IColumnValuesReader[] primaryKeyReaders;
+ protected final VoidPointable[] primaryKeys;
+ protected final AbstractBytesInputStream[] columnStreams;
+
+ protected AbstractAsterixColumnTupleReference(int componentIndex, ColumnBTreeReadLeafFrame frame,
+ IColumnProjectionInfo info, IColumnReadMultiPageOp multiPageOp) {
+ super(componentIndex, frame, info, multiPageOp);
+ primaryKeyReaders = getPrimaryKeyReaders(info);
+ int numberOfPrimaryKeys = primaryKeyReaders.length;
+
+ this.primaryKeyStreams = new ByteBufferInputStream[numberOfPrimaryKeys];
+ primaryKeysValueGetters = new IValueGetter[numberOfPrimaryKeys];
+ primaryKeys = new VoidPointable[numberOfPrimaryKeys];
+
+ for (int i = 0; i < numberOfPrimaryKeys; i++) {
+ primaryKeyStreams[i] = new ByteBufferInputStream();
+ primaryKeysValueGetters[i] =
+ ValueGetterFactory.INSTANCE.createValueGetter(primaryKeyReaders[i].getTypeTag());
+ primaryKeys[i] = new VoidPointable();
+ }
+
+ this.columnStreams = new AbstractBytesInputStream[info.getNumberOfProjectedColumns()];
+ for (int i = 0; i < columnStreams.length; i++) {
+ if (info.getColumnIndex(i) >= numberOfPrimaryKeys) {
+ columnStreams[i] = new MultiByteBufferInputStream();
+ } else {
+ columnStreams[i] = new ByteBufferInputStream();
+ }
+ }
+ }
+
+ protected abstract IColumnValuesReader[] getPrimaryKeyReaders(IColumnProjectionInfo info);
+
+ @Override
+ protected final void startPrimaryKey(IColumnBufferProvider provider, int startIndex, int ordinal,
+ int numberOfTuples) throws HyracksDataException {
+ ByteBufferInputStream primaryKeyStream = primaryKeyStreams[ordinal];
+ primaryKeyStream.reset(provider);
+ IColumnValuesReader reader = primaryKeyReaders[ordinal];
+ reader.reset(primaryKeyStream, numberOfTuples);
+ reader.skip(startIndex);
+ }
+
+ @Override
+ protected final void onNext() throws HyracksDataException {
+ for (int i = 0; i < primaryKeys.length; i++) {
+ IColumnValuesReader reader = primaryKeyReaders[i];
+ reader.next();
+ primaryKeys[i].set(primaryKeysValueGetters[i].getValue(reader));
+ }
+ }
+
+ @Override
+ public void lastTupleReached() throws HyracksDataException {
+ //Default: noOp
+ }
+
+ @Override
+ public final int getFieldCount() {
+ return primaryKeys.length;
+ }
+
+ @Override
+ public final byte[] getFieldData(int fIdx) {
+ return primaryKeys[fIdx].getByteArray();
+ }
+
+ @Override
+ public final int getFieldStart(int fIdx) {
+ return primaryKeys[fIdx].getStartOffset();
+ }
+
+ @Override
+ public final int getFieldLength(int fIdx) {
+ return primaryKeys[fIdx].getLength();
+ }
+
+ @Override
+ public final int getTupleSize() {
+ return -1;
+ }
+
+ @Override
+ public final boolean isAntimatter() {
+ /*
+ * The primary key cannot be missing, but the actual tuple is missing. There is no need to check other
+ * primary key readers (for composite primary keys). One primary key reader is sufficient to determine if a
+ * tuple is an anti-matter tuple.
+ */
+ return primaryKeyReaders[0].isMissing();
+ }
+
+ @Override
+ public final int compareTo(IColumnTupleIterator o) {
+ AbstractAsterixColumnTupleReference other = (AbstractAsterixColumnTupleReference) o;
+ int compare = 0;
+ for (int i = 0; i < primaryKeys.length && compare == 0; i++) {
+ compare = primaryKeyReaders[i].compareTo(other.primaryKeyReaders[i]);
+ }
+ return compare;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/AssembledTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/AssembledTupleReference.java
new file mode 100644
index 0000000..6659e78
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/AssembledTupleReference.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.tuple;
+
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class AssembledTupleReference implements ITupleReference {
+ private final int fieldCount;
+ private final int[] offsets;
+ private final int[] lengths;
+ private byte[] data;
+
+ public AssembledTupleReference(int fieldCount) {
+ this.fieldCount = fieldCount;
+ offsets = new int[fieldCount];
+ lengths = new int[fieldCount];
+ }
+
+ public ITupleReference reset(ArrayTupleBuilder tb) {
+ data = tb.getByteArray();
+ int[] fieldEndOffsets = tb.getFieldEndOffsets();
+
+ int j = fieldEndOffsets.length - 1;
+ for (int i = fieldCount - 1; i >= 0; i--) {
+ offsets[i] = j == 0 ? 0 : fieldEndOffsets[j - 1];
+ lengths[i] = fieldEndOffsets[j] - offsets[i];
+ j--;
+ }
+ return this;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fieldCount;
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return data;
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ return offsets[fIdx];
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ return lengths[fIdx];
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
new file mode 100644
index 0000000..c10d415
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.tuple;
+
+import java.nio.ByteBuffer;
+
+import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream;
+import org.apache.asterix.column.operation.lsm.merge.IEndOfPageCallBack;
+import org.apache.asterix.column.operation.lsm.merge.MergeColumnReadMetadata;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+
+public final class MergeColumnTupleReference extends AbstractAsterixColumnTupleReference {
+ private final IColumnValuesReader[] columnReaders;
+ private int skipCount;
+ private IEndOfPageCallBack endOfPageCallBack;
+
+ public MergeColumnTupleReference(int componentIndex, ColumnBTreeReadLeafFrame frame,
+ MergeColumnReadMetadata columnMetadata, IColumnReadMultiPageOp multiPageOp) {
+ super(componentIndex, frame, columnMetadata, multiPageOp);
+ this.columnReaders = columnMetadata.getColumnReaders();
+ }
+
+ @Override
+ protected IColumnValuesReader[] getPrimaryKeyReaders(IColumnProjectionInfo info) {
+ MergeColumnReadMetadata columnMetadata = (MergeColumnReadMetadata) info;
+ int numberOfPrimaryKeys = columnMetadata.getNumberOfPrimaryKeys();
+ IColumnValuesReader[] primaryKeyReaders = new IColumnValuesReader[numberOfPrimaryKeys];
+ System.arraycopy(columnMetadata.getColumnReaders(), 0, primaryKeyReaders, 0, numberOfPrimaryKeys);
+ return primaryKeyReaders;
+ }
+
+ @Override
+ protected boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples) {
+ //Skip filters
+ pageZero.position(pageZero.position() + numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE);
+ skipCount = 0;
+ return true;
+ }
+
+ @Override
+ protected void startColumn(IColumnBufferProvider buffersProvider, int startIndex, int ordinal, int numberOfTuples)
+ throws HyracksDataException {
+ int numberOfPrimaryKeys = primaryKeys.length;
+ if (ordinal < numberOfPrimaryKeys) {
+ //Skip primary key
+ return;
+ }
+ MultiByteBufferInputStream columnStream = (MultiByteBufferInputStream) columnStreams[ordinal];
+ columnStream.reset(buffersProvider);
+ IColumnValuesReader reader = columnReaders[ordinal];
+ reader.reset(columnStream, numberOfTuples);
+ reader.skip(startIndex);
+ }
+
+ @Override
+ public void skip(int count) throws HyracksDataException {
+ skipCount += count;
+ }
+
+ @Override
+ public void lastTupleReached() throws HyracksDataException {
+ endOfPageCallBack.callEnd(this);
+ }
+
+ public int getAndResetSkipCount() {
+ int currentSkipCount = skipCount;
+ skipCount = 0;
+ return currentSkipCount;
+ }
+
+ public IColumnValuesReader getReader(int columnIndex) {
+ return columnReaders[columnIndex];
+ }
+
+ public void registerEndOfPageCallBack(IEndOfPageCallBack endOfPageCallBack) {
+ this.endOfPageCallBack = endOfPageCallBack;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
new file mode 100644
index 0000000..e286235
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.tuple;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.column.operation.query.ColumnAssembler;
+import org.apache.asterix.column.operation.query.QueryColumnMetadata;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+
+public final class QueryColumnTupleReference extends AbstractAsterixColumnTupleReference {
+ private final ColumnAssembler assembler;
+ private final IColumnFilterEvaluator filterEvaluator;
+ private final List<IColumnFilterValueAccessor> filterValueAccessors;
+
+ public QueryColumnTupleReference(int componentIndex, ColumnBTreeReadLeafFrame frame,
+ QueryColumnMetadata columnMetadata, IColumnReadMultiPageOp multiPageOp) {
+ super(componentIndex, frame, columnMetadata, multiPageOp);
+ assembler = columnMetadata.getAssembler();
+ filterEvaluator = columnMetadata.getFilterEvaluator();
+ filterValueAccessors = columnMetadata.getFilterValueAccessors();
+ }
+
+ @Override
+ protected IColumnValuesReader[] getPrimaryKeyReaders(IColumnProjectionInfo info) {
+ return ((QueryColumnMetadata) info).getPrimaryKeyReaders();
+ }
+
+ @Override
+ protected boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples) {
+ //Skip to filters
+ pageZero.position(pageZero.position() + numberOfColumns * Integer.BYTES);
+ //Set filters' values
+ FilterAccessorProvider.setFilterValues(filterValueAccessors, pageZero, numberOfColumns);
+ //Skip filters
+ pageZero.position(pageZero.position() + numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE);
+ //Check if we should read all column pages
+ boolean readColumns = filterEvaluator.evaluate();
+ assembler.reset(readColumns ? numberOfTuples : 0);
+ return readColumns;
+ }
+
+ @Override
+ protected void startColumn(IColumnBufferProvider buffersProvider, int startIndex, int ordinal, int numberOfTuples)
+ throws HyracksDataException {
+ AbstractBytesInputStream columnStream = columnStreams[ordinal];
+ columnStream.reset(buffersProvider);
+ assembler.resetColumn(columnStream, startIndex, ordinal);
+ }
+
+ @Override
+ public void skip(int count) throws HyracksDataException {
+ assembler.skip(count);
+ }
+
+ public IValueReference getAssembledValue() throws HyracksDataException {
+ return assembler.nextValue();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
new file mode 100644
index 0000000..a5cedc1
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.tuple;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.column.operation.query.ColumnAssembler;
+import org.apache.asterix.column.operation.query.QueryColumnMetadata;
+import org.apache.asterix.column.operation.query.QueryColumnWithMetaMetadata;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+
+public final class QueryColumnWithMetaTupleReference extends AbstractAsterixColumnTupleReference {
+ private final ColumnAssembler assembler;
+ private final ColumnAssembler metaAssembler;
+ private final IColumnFilterEvaluator filterEvaluator;
+ private final List<IColumnFilterValueAccessor> filterValueAccessors;
+
+ public QueryColumnWithMetaTupleReference(int componentIndex, ColumnBTreeReadLeafFrame frame,
+ QueryColumnMetadata columnMetadata, IColumnReadMultiPageOp multiPageOp) {
+ super(componentIndex, frame, columnMetadata, multiPageOp);
+ assembler = columnMetadata.getAssembler();
+ metaAssembler = ((QueryColumnWithMetaMetadata) columnMetadata).getMetaAssembler();
+ filterEvaluator = columnMetadata.getFilterEvaluator();
+ filterValueAccessors = columnMetadata.getFilterValueAccessors();
+ }
+
+ @Override
+ protected IColumnValuesReader[] getPrimaryKeyReaders(IColumnProjectionInfo info) {
+ return ((QueryColumnMetadata) info).getPrimaryKeyReaders();
+ }
+
+ @Override
+ protected boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples) {
+ //Skip to filters
+ pageZero.position(pageZero.position() + numberOfColumns * Integer.BYTES);
+ //Set filters' values
+ FilterAccessorProvider.setFilterValues(filterValueAccessors, pageZero, numberOfColumns);
+ //Skip filters
+ pageZero.position(pageZero.position() + numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE);
+ //Check if we should read all column pages
+ boolean readColumns = filterEvaluator.evaluate();
+ assembler.reset(readColumns ? numberOfTuples : 0);
+ metaAssembler.reset(readColumns ? numberOfTuples : 0);
+ return readColumns;
+ }
+
+ @Override
+ protected void startColumn(IColumnBufferProvider buffersProvider, int startIndex, int ordinal, int numberOfTuples)
+ throws HyracksDataException {
+ AbstractBytesInputStream columnStream = columnStreams[ordinal];
+ columnStream.reset(buffersProvider);
+ int metaColumnCount = metaAssembler.getNumberOfColumns();
+ if (ordinal >= metaColumnCount) {
+ assembler.resetColumn(columnStream, startIndex, ordinal - metaColumnCount);
+ } else {
+ metaAssembler.resetColumn(columnStream, startIndex, ordinal);
+ }
+ }
+
+ @Override
+ public void skip(int count) throws HyracksDataException {
+ metaAssembler.skip(count);
+ assembler.skip(count);
+ }
+
+ public IValueReference getAssembledValue() throws HyracksDataException {
+ return assembler.nextValue();
+ }
+
+ public IValueReference getMetaAssembledValue() throws HyracksDataException {
+ return metaAssembler.nextValue();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
new file mode 100644
index 0000000..0ecdeef
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.util;
+
+public class ColumnValuesUtil {
+ private ColumnValuesUtil() {
+ }
+
+ public static int getBitWidth(int level) {
+ //+1 for the null bit
+ return (32 - Integer.numberOfLeadingZeros(level)) + 1;
+ }
+
+ public static int getNullMask(int level) {
+ return 1 << getBitWidth(level) - 1;
+ }
+
+ public static boolean isNull(int mask, int level) {
+ return (mask & level) == mask;
+ }
+
+ public static int getChildValue(int parentMask, int childMask, int level) {
+ if (isNull(parentMask, level)) {
+ return clearNullBit(parentMask, level) | childMask;
+ }
+ return level;
+ }
+
+ public static int clearNullBit(int nullBitMask, int level) {
+ return (nullBitMask - 1) & level;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java
new file mode 100644
index 0000000..df238cb
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.util;
+
+import java.util.Arrays;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+
+/**
+ * Run-length integer array is to be used for storing repetitive integer values. This is intended for
+ * storing a large number of repeated integers (~1000s). It is not recommended for storing smaller number of integers.
+ * This structure maintains two arrays:
+ * - blockValues: stores the array values
+ * - blockCounts: stores the counts of values in <code>blockValues</code> in a monotonic fashion
+ * <pr>
+ * Example:
+ * Original Array: [1,1,1,1,1,1,2,2,2,1,1,1]
+ * blockValues: [1,2,1]
+ * blockCounts: [6,10,13]
+ */
+public final class RunLengthIntArray {
+ private final IntArrayList blockValues;
+ private int[] blockCounts;
+ private int lastSeen;
+ private int size;
+
+ public RunLengthIntArray() {
+ blockValues = new IntArrayList();
+ blockCounts = new int[32];
+ reset();
+ }
+
+ public void reset() {
+ blockValues.clear();
+ lastSeen = -1;
+ size = 0;
+ }
+
+ public void add(int value) {
+ if (size == 0 || value != lastSeen) {
+ lastSeen = value;
+ newBlock();
+ blockValues.add(value);
+ }
+ blockCounts[blockValues.size() - 1]++;
+ size++;
+ }
+
+ public void add(int value, int count) {
+ if (count == 0) {
+ return;
+ }
+ if (size == 0 || value != lastSeen) {
+ lastSeen = value;
+ newBlock();
+ blockValues.add(value);
+ }
+ blockCounts[blockValues.size() - 1] += count;
+ size += count;
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ public int getNumberOfBlocks() {
+ return blockValues.size();
+ }
+
+ public int getBlockValue(int blockIndex) {
+ return blockValues.getInt(blockIndex);
+ }
+
+ public int getBlockSize(int blockIndex) {
+ if (blockIndex == 0) {
+ return blockCounts[blockIndex];
+ }
+ return blockCounts[blockIndex] - blockCounts[blockIndex - 1];
+ }
+
+ public int getBlockSize(int blockIndex, int startIndex) {
+ return blockCounts[blockIndex] - startIndex;
+ }
+
+ public int getBlockIndex(int startIndex) {
+ if (startIndex >= size) {
+ throw new IndexOutOfBoundsException("startIndex: " + startIndex + " >= size:" + size);
+ }
+ int index = Arrays.binarySearch(blockCounts, 0, blockValues.size(), startIndex);
+ if (index < 0) {
+ index = Math.abs(index) - 1;
+ }
+ return index;
+ }
+
+ public void add(RunLengthIntArray other, int startIndex) {
+ if (startIndex >= other.size) {
+ throw new IndexOutOfBoundsException("startIndex: " + startIndex + " >= other size:" + size);
+ }
+ //First, handle the first block as startIndex might be at the middle of a block
+ //Get which block that startIndex resides
+ int otherBlockIndex = other.getBlockIndex(startIndex);
+ //Get the remaining of the first block starting from startIndex
+ int otherBlockSizeRemaining = other.getBlockSize(otherBlockIndex, startIndex);
+ //Batch add all the remaining values
+ add(other.getBlockValue(otherBlockIndex), otherBlockSizeRemaining);
+
+ //Add other blocks as batches
+ for (int i = otherBlockIndex + 1; i < other.getNumberOfBlocks(); i++) {
+ add(other.getBlockValue(i), other.getBlockSize(i));
+ }
+ }
+
+ private void newBlock() {
+ int newBlockIndex = blockValues.size();
+ if (newBlockIndex == blockCounts.length) {
+ int[] newRepCount = new int[blockCounts.length * 2];
+ System.arraycopy(blockCounts, 0, newRepCount, 0, blockCounts.length);
+ blockCounts = newRepCount;
+ }
+ if (newBlockIndex > 0) {
+ /*
+ * To easily compute where the actual block resides, the block counts are always increasing.
+ * For example:
+ * - Let blockCounts = [5, 6, 13] and blockValues = [1, 0, 1]
+ * - The block sizes are 5, 1, and 7 respectively
+ * - Let say that we want to know what is the value at index 11 by calling getValue(11)
+ * - by searching blockCounts, we know it is at the block with index 2
+ * - Then the value is 1
+ */
+ blockCounts[newBlockIndex] = blockCounts[newBlockIndex - 1];
+ } else {
+ blockCounts[0] = 0;
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (size == 0) {
+ return "[]";
+ }
+ StringBuilder builder = new StringBuilder();
+ int i = 0;
+ builder.append("size: ");
+ builder.append(getSize());
+ builder.append(" [");
+ for (; i < getNumberOfBlocks() - 1; i++) {
+ appendBlockInfo(i, builder);
+ builder.append(',');
+ }
+ appendBlockInfo(i, builder);
+ builder.append(']');
+ return builder.toString();
+ }
+
+ private void appendBlockInfo(int blockIndex, StringBuilder builder) {
+ builder.append('(');
+ builder.append(getBlockValue(blockIndex));
+ builder.append(',');
+ builder.append(getBlockSize(blockIndex));
+ builder.append(')');
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnBatchWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnBatchWriter.java
new file mode 100644
index 0000000..fc1173f
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnBatchWriter.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values;
+
+import java.nio.ByteBuffer;
+import java.util.PriorityQueue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IColumnBatchWriter {
+ void setPageZeroBuffer(ByteBuffer pageZeroBuffer, int numberOfColumns, int numberOfPrimaryKeys);
+
+ /**
+ * Writes the primary keys' values to Page0
+ *
+ * @param primaryKeyWriters primary keys' writers
+ * @return the allocated space for the primary keys' writers
+ */
+ int writePrimaryKeyColumns(IColumnValuesWriter[] primaryKeyWriters) throws HyracksDataException;
+
+ /**
+ * Writes the non-key values to multiple pages
+ *
+ * @param nonKeysColumnWriters non-key values' writers
+ * @return the allocated space for the non-key values' writers
+ */
+ int writeColumns(PriorityQueue<IColumnValuesWriter> nonKeysColumnWriters) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesReader.java
new file mode 100644
index 0000000..0f4cc0c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesReader.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public interface IColumnValuesReader extends Comparable<IColumnValuesReader> {
+ /**
+ * Reset the reader
+ *
+ * @param in input stream that contains the values
+ * @param tupleCount tuple count this column batch belongs to
+ */
+ void reset(AbstractBytesInputStream in, int tupleCount) throws HyracksDataException;
+
+ /* ***********************
+ * Iteration functions
+ * ***********************
+ */
+
+ /**
+ * Move the next value
+ *
+ * @return true if next value was found, false if the end of the values
+ */
+ boolean next() throws HyracksDataException;
+
+ /* ***********************
+ * Information functions
+ * ***********************
+ */
+ ATypeTag getTypeTag();
+
+ /**
+ * @return columnIndex
+ */
+ int getColumnIndex();
+
+ /**
+ * @return Level of the value, which determines if it is NULL, MISSING, or VALUE
+ */
+ int getLevel();
+
+ /**
+ * @return is the current value MISSING
+ */
+ boolean isMissing();
+
+ /**
+ * @return is the current value NULL
+ */
+ boolean isNull();
+
+ /**
+ * @return is an actual value (i.e., neither NULL or MISSING)
+ */
+ boolean isValue();
+
+ /**
+ * @return is this column belongs to an array or multiset
+ */
+ boolean isRepeated();
+
+ /**
+ * @return is it an end of an array (arrays could be nested, and we can hit different delimiters)
+ */
+ boolean isDelimiter();
+
+ /**
+ * @return which delimiter was returned (nested arrays have different delimiter indexes)
+ */
+ int getDelimiterIndex();
+
+ /* ***********************
+ * Value functions
+ * ***********************
+ */
+
+ long getLong();
+
+ double getDouble();
+
+ boolean getBoolean();
+
+ IValueReference getBytes();
+
+ /* ***********************
+ * Write function
+ * ***********************
+ */
+
+ /**
+ * Write the content of reader to the writer
+ *
+ * @param writer to which is the content of this reader is written to
+ * @param callNext should call next on write
+ */
+ void write(IColumnValuesWriter writer, boolean callNext) throws HyracksDataException;
+
+ /**
+ * Write the content of reader to the writer
+ *
+ * @param writer to which is the content of this reader is written to
+ * @param count number of values to write
+ */
+ void write(IColumnValuesWriter writer, int count) throws HyracksDataException;
+
+ /**
+ * Skip values
+ *
+ * @param count the number of values should be skipped
+ */
+ void skip(int count) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesReaderFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesReaderFactory.java
new file mode 100644
index 0000000..7c41512
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesReaderFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values;
+
+import java.io.DataInput;
+import java.io.IOException;
+
+import org.apache.asterix.column.metadata.PathInfoSerializer;
+import org.apache.asterix.om.types.ATypeTag;
+
+public interface IColumnValuesReaderFactory {
+ /**
+ * Create reader for a non-repeated primitive type
+ *
+ * @param typeTag primitive type tag
+ * @param columnIndex column index
+ * @param maxLevel maximum definition levels
+ * @param primaryKey is the value belongs to a primary key?
+ * @return columnar reader
+ */
+ IColumnValuesReader createValueReader(ATypeTag typeTag, int columnIndex, int maxLevel, boolean primaryKey);
+
+ /**
+ * Create a reader for a repeated primitive type
+ *
+ * @param typeTag primitive type tag
+ * @param columnIndex column index
+ * @param maxLevel maximum definition levels
+ * @param delimiters the definition levels for array delimiters
+ * @return columnar reader
+ */
+ IColumnValuesReader createValueReader(ATypeTag typeTag, int columnIndex, int maxLevel, int[] delimiters);
+
+ /**
+ * Create a reader from a serialized path info
+ *
+ * @param input column metadata info
+ * @return columnar reader
+ * @see PathInfoSerializer#writePathInfo(ATypeTag, int, boolean) for more information on how the path info is serialized
+ */
+ IColumnValuesReader createValueReader(DataInput input) throws IOException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriter.java
new file mode 100644
index 0000000..d4e6099
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriter.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+/**
+ * Column writer for values
+ */
+public interface IColumnValuesWriter {
+
+ /**
+ * Reset the writer
+ */
+ void reset() throws HyracksDataException;
+
+ /**
+ * @return the corresponding index of a column
+ */
+ int getColumnIndex();
+
+ /**
+ * Write a value that are not MISSING or NULL
+ *
+ * @param tag value type tag
+ * @param value value reference
+ */
+ void writeValue(ATypeTag tag, IValueReference value) throws HyracksDataException;
+
+ /**
+ * Writing an anti-matter primary key value
+ *
+ * @param value value reference
+ */
+ void writeAntiMatter(ATypeTag tag, IValueReference value) throws HyracksDataException;
+
+ /**
+ * Write level
+ *
+ * @param level level of the value
+ */
+ void writeLevel(int level) throws HyracksDataException;
+
+ /**
+ * Convenient way to write a level multiple times
+ *
+ * @param level level of the value
+ * @param count the number of level occurrences
+ */
+ void writeLevels(int level, int count) throws HyracksDataException;
+
+ /**
+ * For all writers except for {@link ATypeTag#NULL} writer, this method will return null
+ *
+ * @return the definition levels if this is a {@link ATypeTag#NULL} writer, {@code null} otherwise
+ */
+ RunLengthIntArray getDefinitionLevelsIntArray();
+
+ /**
+ * Write NULL
+ *
+ * @param level at what level the NULL occurred
+ */
+ void writeNull(int level) throws HyracksDataException;
+
+ /**
+ * Write a non-unknown value from a reader. Not intended for writing {@link ATypeTag#NULL} or
+ * {@link ATypeTag#MISSING}
+ */
+ void writeValue(IColumnValuesReader reader) throws HyracksDataException;
+
+ /**
+ * @return (probably) an overestimated size of the encoded values
+ */
+ int getEstimatedSize();
+
+ /**
+ * @return the allocated space in bytes
+ */
+ int getAllocatedSpace();
+
+ /**
+ * @return the total count of values
+ */
+ int getCount();
+
+ /**
+ * @return normalized minimum column value
+ */
+ long getNormalizedMinValue();
+
+ /**
+ * @return normalized maximum column value
+ */
+ long getNormalizedMaxValue();
+
+ /**
+ * Flush the columns value to output stream
+ *
+ * @param out output stream
+ */
+ void flush(OutputStream out) throws HyracksDataException;
+
+ /**
+ * Close the writer and release all allocated buffers
+ */
+ void close();
+
+ /**
+ * Serialize the writer
+ *
+ * @param output destination to which the writer should be serialized to
+ */
+ void serialize(DataOutput output) throws IOException;
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriterFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriterFactory.java
new file mode 100644
index 0000000..c858376
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriterFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values;
+
+import org.apache.asterix.om.types.ATypeTag;
+
+public interface IColumnValuesWriterFactory {
+ /**
+ * Create a writer
+ *
+ * @param tag column type
+ * @param columnIndex column index
+ * @param level maximum level that determine a value is not null or missing
+ * @param writeAlways should writer always despite the fact all values were missing/null
+ * @param filtered has a column filter
+ * @return a writer
+ */
+ IColumnValuesWriter createValueWriter(ATypeTag tag, int columnIndex, int level, boolean writeAlways,
+ boolean filtered);
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
new file mode 100644
index 0000000..c0cf18a
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.decoder.ParquetRunLengthBitPackingHybridDecoder;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream;
+import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream;
+import org.apache.asterix.column.util.ColumnValuesUtil;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.reader.value.AbstractValueReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.bytes.BytesUtils;
+
+abstract class AbstractColumnValuesReader implements IColumnValuesReader {
+ protected final AbstractValueReader valueReader;
+ protected final int columnIndex;
+ protected final int maxLevel;
+ protected final ParquetRunLengthBitPackingHybridDecoder definitionLevels;
+ protected final AbstractBytesInputStream valuesStream;
+ protected int level;
+ protected int valueCount;
+ protected int valueIndex;
+
+ private int nullBitMask;
+ private boolean nullLevel;
+ private boolean allMissing;
+
+ AbstractColumnValuesReader(AbstractValueReader valueReader, int columnIndex, int maxLevel, boolean primaryKey) {
+ this.valueReader = valueReader;
+ this.columnIndex = columnIndex;
+ this.maxLevel = maxLevel;
+ definitionLevels = new ParquetRunLengthBitPackingHybridDecoder(ColumnValuesUtil.getBitWidth(maxLevel));
+ valuesStream = primaryKey ? new ByteBufferInputStream() : new MultiByteBufferInputStream();
+ }
+
+ final void nextLevel() throws HyracksDataException {
+ if (allMissing) {
+ return;
+ }
+ try {
+ int actualLevel = definitionLevels.readInt();
+ //Check whether the level is for a null value
+ nullLevel = ColumnValuesUtil.isNull(nullBitMask, actualLevel);
+ //Clear the null bit to allow repeated value readers determine the correct delimiter for null values
+ level = ColumnValuesUtil.clearNullBit(nullBitMask, actualLevel);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ abstract void resetValues();
+
+ @Override
+ public final void reset(AbstractBytesInputStream in, int tupleCount) throws HyracksDataException {
+ valueIndex = 0;
+ if (in.available() == 0) {
+ allMissing = true;
+ level = 0;
+ valueCount = tupleCount;
+ return;
+ }
+ allMissing = false;
+ try {
+ nullBitMask = ColumnValuesUtil.getNullMask(BytesUtils.readZigZagVarInt(in));
+ int defLevelsSize = BytesUtils.readZigZagVarInt(in);
+ valueCount = BytesUtils.readZigZagVarInt(in);
+ definitionLevels.reset(in);
+ valuesStream.resetAt(defLevelsSize, in);
+ int valueLength = BytesUtils.readZigZagVarInt(valuesStream);
+ if (valueLength > 0) {
+ valueReader.resetValue(valuesStream);
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ resetValues();
+ }
+
+ @Override
+ public final ATypeTag getTypeTag() {
+ return valueReader.getTypeTag();
+ }
+
+ @Override
+ public final int getColumnIndex() {
+ return columnIndex;
+ }
+
+ @Override
+ public int getLevel() {
+ return level;
+ }
+
+ @Override
+ public final boolean isMissing() {
+ return !isDelimiter() && level < maxLevel;
+ }
+
+ @Override
+ public final boolean isNull() {
+ return nullLevel;
+ }
+
+ @Override
+ public final boolean isValue() {
+ return !isNull() && level == maxLevel;
+ }
+
+ @Override
+ public final long getLong() {
+ return valueReader.getLong();
+ }
+
+ @Override
+ public final double getDouble() {
+ return valueReader.getDouble();
+ }
+
+ @Override
+ public final boolean getBoolean() {
+ return valueReader.getBoolean();
+ }
+
+ @Override
+ public final IValueReference getBytes() {
+ return valueReader.getBytes();
+ }
+
+ @Override
+ public final int compareTo(IColumnValuesReader o) {
+ return valueReader.compareTo(((AbstractColumnValuesReader) o).valueReader);
+ }
+
+ @Override
+ public final void write(IColumnValuesWriter writer, int count) throws HyracksDataException {
+ for (int i = 0; i < count; i++) {
+ write(writer, true);
+ }
+ }
+
+ @Override
+ public void skip(int count) throws HyracksDataException {
+ for (int i = 0; i < count; i++) {
+ next();
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/ColumnValueReaderFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/ColumnValueReaderFactory.java
new file mode 100644
index 0000000..b233482
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/ColumnValueReaderFactory.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader;
+
+import java.io.DataInput;
+import java.io.IOException;
+
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesReaderFactory;
+import org.apache.asterix.column.values.reader.value.AbstractValueReader;
+import org.apache.asterix.column.values.reader.value.BooleanValueReader;
+import org.apache.asterix.column.values.reader.value.DoubleValueReader;
+import org.apache.asterix.column.values.reader.value.LongValueReader;
+import org.apache.asterix.column.values.reader.value.NoOpValueReader;
+import org.apache.asterix.column.values.reader.value.StringValueReader;
+import org.apache.asterix.column.values.reader.value.UUIDValueReader;
+import org.apache.asterix.om.types.ATypeTag;
+
+public class ColumnValueReaderFactory implements IColumnValuesReaderFactory {
+ @Override
+ public IColumnValuesReader createValueReader(ATypeTag typeTag, int columnIndex, int maxLevel, boolean primaryKey) {
+ return new PrimitiveColumnValuesReader(createReader(typeTag), columnIndex, maxLevel, primaryKey);
+ }
+
+ @Override
+ public IColumnValuesReader createValueReader(ATypeTag typeTag, int columnIndex, int maxLevel, int[] delimiters) {
+ return new RepeatedPrimitiveColumnValuesReader(createReader(typeTag), columnIndex, maxLevel, delimiters);
+ }
+
+ @Override
+ public IColumnValuesReader createValueReader(DataInput input) throws IOException {
+ ATypeTag typeTag = ATypeTag.VALUE_TYPE_MAPPING[input.readByte()];
+ int columnIndex = input.readInt();
+ int maxLevel = input.readInt();
+ boolean primaryKey = input.readBoolean();
+ boolean collection = input.readBoolean();
+ if (collection) {
+ int[] delimiters = new int[input.readInt()];
+ for (int i = 0; i < delimiters.length; i++) {
+ delimiters[i] = input.readInt();
+ }
+ return createValueReader(typeTag, columnIndex, maxLevel, delimiters);
+ }
+ return createValueReader(typeTag, columnIndex, maxLevel, primaryKey);
+ }
+
+ private AbstractValueReader createReader(ATypeTag typeTag) {
+ switch (typeTag) {
+ case MISSING:
+ case NULL:
+ return NoOpValueReader.INSTANCE;
+ case BOOLEAN:
+ return new BooleanValueReader();
+ case BIGINT:
+ return new LongValueReader();
+ case DOUBLE:
+ return new DoubleValueReader();
+ case STRING:
+ return new StringValueReader();
+ case UUID:
+ return new UUIDValueReader();
+ default:
+ throw new UnsupportedOperationException(typeTag + " is not supported");
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java
new file mode 100644
index 0000000..e8c7bc5
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.reader.value.AbstractValueReader;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Reader for a non-repeated primitive value
+ */
+public final class PrimitiveColumnValuesReader extends AbstractColumnValuesReader {
+ /**
+ * A primary key value is always present. Anti-matter can be determined by checking whether the definition level
+ * indicates that the tuple's values are missing (i.e., by calling {@link #isMissing()}).
+ */
+ private final boolean primaryKey;
+
+ public PrimitiveColumnValuesReader(AbstractValueReader reader, int columnIndex, int maxLevel, boolean primaryKey) {
+ super(reader, columnIndex, maxLevel, primaryKey);
+ this.primaryKey = primaryKey;
+ }
+
+ @Override
+ public void resetValues() {
+ //NoOp
+ }
+
+ @Override
+ public boolean next() throws HyracksDataException {
+ if (valueIndex == valueCount) {
+ return false;
+ }
+ valueIndex++;
+
+ try {
+ nextLevel();
+ if (primaryKey || level == maxLevel) {
+ valueReader.nextValue();
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean isRepeated() {
+ return false;
+ }
+
+ @Override
+ public boolean isDelimiter() {
+ return false;
+ }
+
+ @Override
+ public int getDelimiterIndex() {
+ throw new IllegalStateException("Not a repeated reader");
+ }
+
+ @Override
+ public void write(IColumnValuesWriter writer, boolean callNext) throws HyracksDataException {
+ if (callNext && !next()) {
+ throw new IllegalStateException("No more values");
+ }
+
+ writer.writeLevel(level);
+ if (primaryKey || isValue()) {
+ try {
+ writer.writeValue(this);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java
new file mode 100644
index 0000000..0fb98be
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.reader.value.AbstractValueReader;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * For primitive values that belong to an {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET}
+ */
+public final class RepeatedPrimitiveColumnValuesReader extends AbstractColumnValuesReader {
+ private final int[] delimiters;
+ private final int[] levelToDelimiterMap;
+ private int delimiterIndex;
+
+ RepeatedPrimitiveColumnValuesReader(AbstractValueReader valueReader, int columnIndex, int maxLevel,
+ int[] delimiters) {
+ super(valueReader, columnIndex, maxLevel, false);
+ this.delimiters = delimiters;
+ delimiterIndex = delimiters.length;
+
+ levelToDelimiterMap = new int[maxLevel + 1];
+ int currentDelimiterIndex = 0;
+ for (int level = maxLevel; level >= 0; level--) {
+ if (currentDelimiterIndex < delimiters.length && level == delimiters[currentDelimiterIndex]) {
+ currentDelimiterIndex++;
+ }
+ levelToDelimiterMap[level] = currentDelimiterIndex;
+ }
+ }
+
+ @Override
+ protected void resetValues() {
+ delimiterIndex = delimiters.length;
+ }
+
+ @Override
+ public boolean next() throws HyracksDataException {
+ if (valueIndex == valueCount) {
+ return false;
+ }
+
+ consumeDelimiterIfAny();
+ nextLevel();
+ setDelimiterIndex();
+ if (level == maxLevel) {
+ valueReader.nextValue();
+ }
+ valueIndex++;
+ return true;
+ }
+
+ @Override
+ public boolean isRepeated() {
+ return true;
+ }
+
+ @Override
+ public boolean isDelimiter() {
+ return delimiterIndex < delimiters.length && level == delimiters[delimiterIndex];
+ }
+
+ @Override
+ public int getDelimiterIndex() {
+ return delimiterIndex;
+ }
+
+ @Override
+ public void write(IColumnValuesWriter writer, boolean callNext) throws HyracksDataException {
+ //We always call next as repeated values cannot be primary keys
+ if (!next()) {
+ throw new IllegalStateException("No more values");
+ }
+
+ if (isRepeatedValue()) {
+ while (!isLastDelimiter()) {
+ writer.writeLevel(level);
+ if (isValue()) {
+ try {
+ writer.writeValue(this);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ next();
+ }
+ }
+ //Add last delimiter, or NULL/MISSING
+ writer.writeLevel(level);
+ }
+
+ private boolean isRepeatedValue() {
+ return levelToDelimiterMap[level] < delimiters.length;
+ }
+
+ private boolean isLastDelimiter() {
+ return isDelimiter() && delimiterIndex == delimiters.length - 1;
+ }
+
+ private void consumeDelimiterIfAny() {
+ if (isDelimiter()) {
+ delimiterIndex++;
+ }
+ }
+
+ private void setDelimiterIndex() {
+ if (isDelimiter() || level <= delimiters[delimiters.length - 1]) {
+ return;
+ }
+ delimiterIndex = levelToDelimiterMap[level];
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/FilterAccessorProvider.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/FilterAccessorProvider.java
new file mode 100644
index 0000000..2c8cf19
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/FilterAccessorProvider.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.column.metadata.schema.visitor.PathExtractorVisitor;
+import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
+import org.apache.asterix.column.values.reader.filter.value.ColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.value.NoOpColumnFilterValueAccessor;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FilterAccessorProvider {
+ private final ObjectSchemaNode root;
+ private final ObjectSchemaNode metaRoot;
+ private final SchemaClipperVisitor clipperVisitor;
+ private final PathExtractorVisitor pathExtractorVisitor;
+ private final Map<ARecordType, PrimitiveSchemaNode> cachedNodes;
+ private final List<IColumnFilterValueAccessor> filterAccessors;
+
+ public FilterAccessorProvider(ObjectSchemaNode root, SchemaClipperVisitor clipperVisitor) {
+ this(root, null, clipperVisitor);
+ }
+
+ public FilterAccessorProvider(ObjectSchemaNode root, ObjectSchemaNode metaRoot,
+ SchemaClipperVisitor clipperVisitor) {
+ this.root = root;
+ this.metaRoot = metaRoot;
+ this.clipperVisitor = clipperVisitor;
+ this.pathExtractorVisitor = new PathExtractorVisitor();
+ cachedNodes = new HashMap<>();
+ filterAccessors = new ArrayList<>();
+ }
+
+ public IColumnFilterValueAccessor createColumnFilterValueAccessor(ARecordType path, boolean min)
+ throws HyracksDataException {
+ PrimitiveSchemaNode node = getNode(path);
+ ATypeTag typeTag = node.getTypeTag();
+ if (typeTag == ATypeTag.MISSING) {
+ return NoOpColumnFilterValueAccessor.INSTANCE;
+ }
+ IColumnFilterValueAccessor accessor = new ColumnFilterValueAccessor(node.getColumnIndex(), typeTag, min);
+ filterAccessors.add(accessor);
+ return accessor;
+ }
+
+ public List<IColumnFilterValueAccessor> getFilterAccessors() {
+ return filterAccessors;
+ }
+
+ public static void setFilterValues(List<IColumnFilterValueAccessor> filterValueAccessors, ByteBuffer pageZero,
+ int numberOfColumns) {
+ for (int i = 0; i < filterValueAccessors.size(); i++) {
+ ColumnFilterValueAccessor accessor = (ColumnFilterValueAccessor) filterValueAccessors.get(i);
+ int columnIndex = accessor.getColumnIndex();
+ long normalizedValue;
+ if (columnIndex < numberOfColumns) {
+ int filterOffset = pageZero.position() + columnIndex * AbstractColumnFilterWriter.FILTER_SIZE;
+ normalizedValue =
+ accessor.isMin() ? pageZero.getLong(filterOffset) : pageZero.getLong(filterOffset + Long.BYTES);
+ } else {
+ // Column is missing
+ normalizedValue = accessor.isMin() ? Long.MAX_VALUE : Long.MIN_VALUE;
+ }
+ accessor.setNormalizedValue(normalizedValue);
+ }
+ }
+
+ private PrimitiveSchemaNode getNode(ARecordType path) throws HyracksDataException {
+ PrimitiveSchemaNode node = cachedNodes.get(path);
+ if (node == null) {
+ ObjectSchemaNode dataPath = (ObjectSchemaNode) path.accept(clipperVisitor, root);
+ node = (PrimitiveSchemaNode) dataPath.accept(pathExtractorVisitor, null);
+ if (node.getTypeTag() == ATypeTag.MISSING && metaRoot != null) {
+ //Try meta
+ ObjectSchemaNode metaPath = (ObjectSchemaNode) path.accept(clipperVisitor, metaRoot);
+ node = (PrimitiveSchemaNode) metaPath.accept(pathExtractorVisitor, null);
+ }
+ cachedNodes.put(path, node);
+ }
+ return node;
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterEvaluator.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterEvaluator.java
index 3c1a24d..bc1f51c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterEvaluator.java
@@ -16,20 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.values.reader.filter;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+public interface IColumnFilterEvaluator {
+ boolean evaluate();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterEvaluatorFactory.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterEvaluatorFactory.java
index 3c1a24d..62abb92 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterEvaluatorFactory.java
@@ -16,20 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.values.reader.filter;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import java.io.Serializable;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+@FunctionalInterface
+public interface IColumnFilterEvaluatorFactory extends Serializable {
+ IColumnFilterEvaluator create(FilterAccessorProvider filterAccessorProvider) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterValueAccessor.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterValueAccessor.java
index 3c1a24d..69abb37 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterValueAccessor.java
@@ -16,20 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.values.reader.filter;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import org.apache.asterix.om.types.ATypeTag;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+public interface IColumnFilterValueAccessor {
+ long getNormalizedValue();
+
+ ATypeTag getTypeTag();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterValueAccessorFactory.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterValueAccessorFactory.java
index 3c1a24d..9621c02 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/IColumnFilterValueAccessorFactory.java
@@ -16,20 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.values.reader.filter;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import java.io.Serializable;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+@FunctionalInterface
+public interface IColumnFilterValueAccessorFactory extends Serializable {
+ IColumnFilterValueAccessor create(FilterAccessorProvider filterAccessorProvider) throws HyracksDataException;
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/AbstractColumnFilterComparatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/AbstractColumnFilterComparatorFactory.java
new file mode 100644
index 0000000..8244876
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/AbstractColumnFilterComparatorFactory.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.compartor;
+
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+import org.apache.asterix.column.values.reader.filter.evaluator.FalseColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.evaluator.TrueColumnFilterEvaluator;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+abstract class AbstractColumnFilterComparatorFactory implements IColumnFilterEvaluatorFactory {
+ private static final long serialVersionUID = 4229059703449173694L;
+ private final IColumnFilterValueAccessorFactory left;
+ private final IColumnFilterValueAccessorFactory right;
+
+ AbstractColumnFilterComparatorFactory(IColumnFilterValueAccessorFactory left,
+ IColumnFilterValueAccessorFactory right) {
+ this.left = left;
+ this.right = right;
+ }
+
+ @Override
+ public final IColumnFilterEvaluator create(FilterAccessorProvider filterAccessorProvider)
+ throws HyracksDataException {
+ IColumnFilterValueAccessor leftAccessor = left.create(filterAccessorProvider);
+ IColumnFilterValueAccessor rightAccessor = right.create(filterAccessorProvider);
+
+ ATypeTag leftTypeTag = leftAccessor.getTypeTag();
+ ATypeTag rightTypeTag = rightAccessor.getTypeTag();
+ if (leftTypeTag != rightTypeTag && ATypeHierarchy.isCompatible(leftTypeTag, rightTypeTag)) {
+ // Cannot compare comparable values with different types. Bail out.
+ return TrueColumnFilterEvaluator.INSTANCE;
+ } else if (cannotCompare(leftTypeTag, rightTypeTag)) {
+ return FalseColumnFilterEvaluator.INSTANCE;
+ }
+ return createComparator(leftAccessor, rightAccessor);
+ }
+
+ private boolean cannotCompare(ATypeTag leftTypeTag, ATypeTag rightTypeTag) {
+ return rightTypeTag == ATypeTag.MISSING || leftTypeTag != rightTypeTag;
+ }
+
+ protected abstract IColumnFilterEvaluator createComparator(IColumnFilterValueAccessor left,
+ IColumnFilterValueAccessor right);
+
+ protected abstract String getOpt();
+
+ @Override
+ public String toString() {
+ return left.toString() + " " + getOpt() + " " + right.toString();
+ }
+
+ static abstract class AbstractComparator implements IColumnFilterEvaluator {
+ protected final IColumnFilterValueAccessor left;
+ protected final IColumnFilterValueAccessor right;
+
+ AbstractComparator(IColumnFilterValueAccessor left, IColumnFilterValueAccessor right) {
+ this.left = left;
+ this.right = right;
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/GEColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/GEColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..81d58db
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/GEColumnFilterEvaluatorFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.compartor;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+
+public class GEColumnFilterEvaluatorFactory extends AbstractColumnFilterComparatorFactory {
+ private static final long serialVersionUID = 6879193736347174789L;
+
+ public GEColumnFilterEvaluatorFactory(IColumnFilterValueAccessorFactory left,
+ IColumnFilterValueAccessorFactory right) {
+ super(left, right);
+ }
+
+ @Override
+ protected IColumnFilterEvaluator createComparator(IColumnFilterValueAccessor left,
+ IColumnFilterValueAccessor right) {
+ return new AbstractComparator(left, right) {
+ @Override
+ public boolean evaluate() {
+ return left.getNormalizedValue() >= right.getNormalizedValue();
+ }
+ };
+ }
+
+ @Override
+ protected String getOpt() {
+ return ">=";
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/GTColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/GTColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..b24e18bf
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/GTColumnFilterEvaluatorFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.compartor;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+
+public class GTColumnFilterEvaluatorFactory extends AbstractColumnFilterComparatorFactory {
+ private static final long serialVersionUID = -3104103170926445020L;
+
+ public GTColumnFilterEvaluatorFactory(IColumnFilterValueAccessorFactory left,
+ IColumnFilterValueAccessorFactory right) {
+ super(left, right);
+ }
+
+ @Override
+ protected IColumnFilterEvaluator createComparator(IColumnFilterValueAccessor left,
+ IColumnFilterValueAccessor right) {
+ return new AbstractComparator(left, right) {
+ @Override
+ public boolean evaluate() {
+ return left.getNormalizedValue() > right.getNormalizedValue();
+ }
+ };
+ }
+
+ @Override
+ protected String getOpt() {
+ return ">";
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/LEColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/LEColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..f195d03
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/LEColumnFilterEvaluatorFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.compartor;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+
+public class LEColumnFilterEvaluatorFactory extends AbstractColumnFilterComparatorFactory {
+ private static final long serialVersionUID = 1068661809768620550L;
+
+ public LEColumnFilterEvaluatorFactory(IColumnFilterValueAccessorFactory left,
+ IColumnFilterValueAccessorFactory right) {
+ super(left, right);
+ }
+
+ @Override
+ protected IColumnFilterEvaluator createComparator(IColumnFilterValueAccessor left,
+ IColumnFilterValueAccessor right) {
+ return new AbstractComparator(left, right) {
+ @Override
+ public boolean evaluate() {
+ return left.getNormalizedValue() <= right.getNormalizedValue();
+ }
+ };
+ }
+
+ @Override
+ protected String getOpt() {
+ return "<=";
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/LTColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/LTColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..41c8018
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/compartor/LTColumnFilterEvaluatorFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.compartor;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+
+public class LTColumnFilterEvaluatorFactory extends AbstractColumnFilterComparatorFactory {
+ private static final long serialVersionUID = -4066709771630858677L;
+
+ public LTColumnFilterEvaluatorFactory(IColumnFilterValueAccessorFactory left,
+ IColumnFilterValueAccessorFactory right) {
+ super(left, right);
+ }
+
+ @Override
+ protected IColumnFilterEvaluator createComparator(IColumnFilterValueAccessor left,
+ IColumnFilterValueAccessor right) {
+ return new AbstractComparator(left, right) {
+ @Override
+ public boolean evaluate() {
+ return left.getNormalizedValue() < right.getNormalizedValue();
+ }
+ };
+ }
+
+ @Override
+ protected String getOpt() {
+ return "<";
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/ANDColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/ANDColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..03ae64e
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/ANDColumnFilterEvaluatorFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.evaluator;
+
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class ANDColumnFilterEvaluatorFactory extends AbstractColumnFilterEvaluatorFactory {
+ private static final long serialVersionUID = -7902069740719309586L;
+
+ public ANDColumnFilterEvaluatorFactory(IColumnFilterEvaluatorFactory left, IColumnFilterEvaluatorFactory right) {
+ super(left, right);
+ }
+
+ @Override
+ public IColumnFilterEvaluator create(FilterAccessorProvider filterAccessorProvider) throws HyracksDataException {
+ IColumnFilterEvaluator leftEval = left.create(filterAccessorProvider);
+ IColumnFilterEvaluator rightEval = right.create(filterAccessorProvider);
+ if (leftEval == FalseColumnFilterEvaluator.INSTANCE || rightEval == FalseColumnFilterEvaluator.INSTANCE) {
+ // Either is false, then return false
+ return FalseColumnFilterEvaluator.INSTANCE;
+ } else if (leftEval == TrueColumnFilterEvaluator.INSTANCE && rightEval == TrueColumnFilterEvaluator.INSTANCE) {
+ //Skip both operands and return TrueColumnFilterEvaluator
+ return TrueColumnFilterEvaluator.INSTANCE;
+ } else if (leftEval == TrueColumnFilterEvaluator.INSTANCE) {
+ //Left is true, return the right evaluator
+ return rightEval;
+ } else if (rightEval == TrueColumnFilterEvaluator.INSTANCE) {
+ //Same as above but the right is true
+ return leftEval;
+ } else {
+ //Both are actual filters
+ return create(leftEval, rightEval);
+ }
+ }
+
+ private IColumnFilterEvaluator create(IColumnFilterEvaluator left, IColumnFilterEvaluator right) {
+ return new AbstractEvaluator(left, right) {
+ @Override
+ public boolean evaluate() {
+ return left.evaluate() && right.evaluate();
+ }
+ };
+ }
+
+ @Override
+ protected String getOp() {
+ return "&&";
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/AbstractColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/AbstractColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..d1a53d1
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/AbstractColumnFilterEvaluatorFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.evaluator;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+
+public abstract class AbstractColumnFilterEvaluatorFactory implements IColumnFilterEvaluatorFactory {
+ private static final long serialVersionUID = 1436531448052787426L;
+
+ protected final IColumnFilterEvaluatorFactory left;
+ protected final IColumnFilterEvaluatorFactory right;
+
+ public AbstractColumnFilterEvaluatorFactory(IColumnFilterEvaluatorFactory left,
+ IColumnFilterEvaluatorFactory right) {
+ this.left = left;
+ this.right = right;
+ }
+
+ protected abstract String getOp();
+
+ @Override
+ public String toString() {
+ return left.toString() + " " + getOp() + " " + right.toString();
+ }
+
+ static abstract class AbstractEvaluator implements IColumnFilterEvaluator {
+ protected final IColumnFilterEvaluator left;
+ protected final IColumnFilterEvaluator right;
+
+ AbstractEvaluator(IColumnFilterEvaluator left, IColumnFilterEvaluator right) {
+ this.left = left;
+ this.right = right;
+ }
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/FalseColumnFilterEvaluator.java
similarity index 60%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/FalseColumnFilterEvaluator.java
index 3c1a24d..fccd015 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/FalseColumnFilterEvaluator.java
@@ -16,20 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.asterix.column.values.reader.filter.evaluator;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+public class FalseColumnFilterEvaluator implements IColumnFilterEvaluator {
+ public static final IColumnFilterEvaluator INSTANCE = new FalseColumnFilterEvaluator();
+
+ private FalseColumnFilterEvaluator() {
+ }
+
+ @Override
+ public boolean evaluate() {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "FALSE";
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/NoOpColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/NoOpColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..765f3c6
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/NoOpColumnFilterEvaluatorFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.evaluator;
+
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class NoOpColumnFilterEvaluatorFactory implements IColumnFilterEvaluatorFactory {
+ private static final long serialVersionUID = -7122361396576592000L;
+ public static final IColumnFilterEvaluatorFactory INSTANCE = new NoOpColumnFilterEvaluatorFactory();
+
+ private NoOpColumnFilterEvaluatorFactory() {
+ }
+
+ @Override
+ public IColumnFilterEvaluator create(FilterAccessorProvider filterAccessorProvider) throws HyracksDataException {
+ // True is also NoOp
+ return TrueColumnFilterEvaluator.INSTANCE;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/ORColumnFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/ORColumnFilterEvaluatorFactory.java
new file mode 100644
index 0000000..aedc9f4
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/ORColumnFilterEvaluatorFactory.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.evaluator;
+
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class ORColumnFilterEvaluatorFactory extends AbstractColumnFilterEvaluatorFactory {
+ private static final long serialVersionUID = 9029706131191375500L;
+
+ public ORColumnFilterEvaluatorFactory(IColumnFilterEvaluatorFactory left, IColumnFilterEvaluatorFactory right) {
+ super(left, right);
+ }
+
+ @Override
+ public IColumnFilterEvaluator create(FilterAccessorProvider filterAccessorProvider) throws HyracksDataException {
+ IColumnFilterEvaluator leftEval = left.create(filterAccessorProvider);
+ IColumnFilterEvaluator rightEval = right.create(filterAccessorProvider);
+ if (leftEval == TrueColumnFilterEvaluator.INSTANCE || rightEval == TrueColumnFilterEvaluator.INSTANCE) {
+ // Either is true, then return true
+ return TrueColumnFilterEvaluator.INSTANCE;
+ } else if (leftEval == FalseColumnFilterEvaluator.INSTANCE
+ && rightEval == FalseColumnFilterEvaluator.INSTANCE) {
+ // Both are false, then return false
+ return FalseColumnFilterEvaluator.INSTANCE;
+ } else if (leftEval == FalseColumnFilterEvaluator.INSTANCE) {
+ //Left is NoOp, which evaluates to true, return the right evaluator
+ return rightEval;
+ } else if (rightEval == FalseColumnFilterEvaluator.INSTANCE) {
+ //Same as above but the right is NoOp
+ return leftEval;
+ } else {
+ //Both are actual evaluators
+ return create(leftEval, rightEval);
+ }
+ }
+
+ private IColumnFilterEvaluator create(IColumnFilterEvaluator left, IColumnFilterEvaluator right) {
+ return new AbstractEvaluator(left, right) {
+ @Override
+ public boolean evaluate() {
+ return left.evaluate() || right.evaluate();
+ }
+ };
+ }
+
+ @Override
+ protected String getOp() {
+ return "||";
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/TrueColumnFilterEvaluator.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/TrueColumnFilterEvaluator.java
new file mode 100644
index 0000000..37f1c5c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/evaluator/TrueColumnFilterEvaluator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.evaluator;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluator;
+
+/**
+ * This evaluator is also used to indicate a NoOp filter
+ */
+public class TrueColumnFilterEvaluator implements IColumnFilterEvaluator {
+ public static final IColumnFilterEvaluator INSTANCE = new TrueColumnFilterEvaluator();
+
+ private TrueColumnFilterEvaluator() {
+ }
+
+ @Override
+ public boolean evaluate() {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "TRUE";
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ColumnFilterValueAccessor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ColumnFilterValueAccessor.java
new file mode 100644
index 0000000..0aa2b31
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ColumnFilterValueAccessor.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.value;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.om.types.ATypeTag;
+
+public class ColumnFilterValueAccessor implements IColumnFilterValueAccessor {
+ private final int columnIndex;
+ private final ATypeTag typeTag;
+ private final boolean min;
+ private long normalizedValue;
+
+ public ColumnFilterValueAccessor(int columnIndex, ATypeTag typeTag, boolean min) {
+ this.columnIndex = columnIndex;
+ this.typeTag = typeTag;
+ this.min = min;
+ }
+
+ public int getColumnIndex() {
+ return columnIndex;
+ }
+
+ public boolean isMin() {
+ return min;
+ }
+
+ public void setNormalizedValue(long normalizedValue) {
+ this.normalizedValue = normalizedValue;
+ }
+
+ @Override
+ public long getNormalizedValue() {
+ return normalizedValue;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return typeTag;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ColumnFilterValueAccessorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ColumnFilterValueAccessorFactory.java
new file mode 100644
index 0000000..fb02321
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ColumnFilterValueAccessorFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.value;
+
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.visitor.PathStringBuilderForIATypeVisitor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class ColumnFilterValueAccessorFactory implements IColumnFilterValueAccessorFactory {
+ private static final long serialVersionUID = -6341611172763952841L;
+ private final ARecordType path;
+ private final boolean min;
+
+ public ColumnFilterValueAccessorFactory(ARecordType path, boolean min) {
+ this.path = path;
+ this.min = min;
+ }
+
+ @Override
+ public IColumnFilterValueAccessor create(FilterAccessorProvider filterAccessorProvider)
+ throws HyracksDataException {
+ return filterAccessorProvider.createColumnFilterValueAccessor(path, min);
+ }
+
+ @Override
+ public String toString() {
+ PathStringBuilderForIATypeVisitor pathBuilder = new PathStringBuilderForIATypeVisitor();
+ StringBuilder stringBuilder = new StringBuilder();
+
+ stringBuilder.append(min ? "min" : "max");
+ stringBuilder.append('(');
+ path.accept(pathBuilder, stringBuilder);
+ stringBuilder.append(')');
+
+ return stringBuilder.toString();
+ }
+
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ConstantColumnFilterValueAccessor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ConstantColumnFilterValueAccessor.java
new file mode 100644
index 0000000..db9ef9c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ConstantColumnFilterValueAccessor.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.value;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.om.types.ATypeTag;
+
+public final class ConstantColumnFilterValueAccessor implements IColumnFilterValueAccessor {
+ private final long normalizedValue;
+ private final ATypeTag typeTag;
+
+ //TODO add UUID
+
+ public ConstantColumnFilterValueAccessor(long normalizedValue, ATypeTag typeTag) {
+ this.normalizedValue = normalizedValue;
+ this.typeTag = typeTag;
+ }
+
+ @Override
+ public long getNormalizedValue() {
+ return normalizedValue;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return typeTag;
+ }
+
+ @Override
+ public String toString() {
+ return Long.toString(normalizedValue);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ConstantColumnFilterValueAccessorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ConstantColumnFilterValueAccessorFactory.java
new file mode 100644
index 0000000..eeb903d
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/ConstantColumnFilterValueAccessorFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.value;
+
+import org.apache.asterix.column.values.reader.filter.FilterAccessorProvider;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+import org.apache.asterix.om.base.ADouble;
+import org.apache.asterix.om.base.AInt64;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class ConstantColumnFilterValueAccessorFactory implements IColumnFilterValueAccessorFactory {
+ private static final long serialVersionUID = -4835407779342615453L;
+ private final long normalizedValue;
+ private final ATypeTag typeTag;
+ private final String stringValue;
+
+ private ConstantColumnFilterValueAccessorFactory(String stringValue, long normalizedValue, ATypeTag typeTag) {
+ this.stringValue = stringValue;
+ this.normalizedValue = normalizedValue;
+ this.typeTag = typeTag;
+ }
+
+ public static ConstantColumnFilterValueAccessorFactory createFactory(IAObject value) {
+ String stringValue;
+ long normalizedValue;
+ ATypeTag typeTag = value.getType().getTypeTag();
+ switch (typeTag) {
+ case BIGINT:
+ long longVal = ((AInt64) value).getLongValue();
+ stringValue = Long.toString(longVal);
+ normalizedValue = longVal;
+ break;
+ case DOUBLE:
+ double doubleVal = ((ADouble) value).getDoubleValue();
+ stringValue = Double.toString(doubleVal);
+ normalizedValue = Double.doubleToLongBits(doubleVal);
+ break;
+ case STRING:
+ stringValue = ((AString) value).getStringValue();
+ normalizedValue = normalize(stringValue);
+ break;
+ default:
+ return null;
+ }
+
+ return new ConstantColumnFilterValueAccessorFactory(stringValue, normalizedValue, typeTag);
+ }
+
+ @Override
+ public IColumnFilterValueAccessor create(FilterAccessorProvider filterAccessorProvider)
+ throws HyracksDataException {
+ return new ConstantColumnFilterValueAccessor(normalizedValue, typeTag);
+ }
+
+ @Override
+ public String toString() {
+ if (typeTag == ATypeTag.STRING) {
+ return "\"" + stringValue + "\"";
+ }
+ return stringValue;
+ }
+
+ private static long normalize(String value) {
+ long nk = 0;
+ for (int i = 0; i < 4; ++i) {
+ nk <<= 16;
+ if (i < value.length()) {
+ nk += value.charAt(i) & 0xffff;
+ }
+ }
+ //Make it always positive
+ return nk >>> 1;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/NoOpColumnFilterValueAccessor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/NoOpColumnFilterValueAccessor.java
new file mode 100644
index 0000000..c52eeda
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/filter/value/NoOpColumnFilterValueAccessor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.filter.value;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessor;
+import org.apache.asterix.om.types.ATypeTag;
+
+public class NoOpColumnFilterValueAccessor implements IColumnFilterValueAccessor {
+ public static final IColumnFilterValueAccessor INSTANCE = new NoOpColumnFilterValueAccessor();
+
+ private NoOpColumnFilterValueAccessor() {
+ }
+
+ @Override
+ public long getNormalizedValue() {
+ throw new IllegalStateException("should not be invoked");
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.MISSING;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/AbstractValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/AbstractValueReader.java
new file mode 100644
index 0000000..3d4c744
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/AbstractValueReader.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.value;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public abstract class AbstractValueReader implements Comparable<AbstractValueReader> {
+
+ public abstract void resetValue(AbstractBytesInputStream in) throws IOException;
+
+ public abstract void nextValue() throws HyracksDataException;
+
+ public abstract ATypeTag getTypeTag();
+
+ public boolean getBoolean() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public long getLong() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public double getDouble() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public IValueReference getBytes() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/BooleanValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/BooleanValueReader.java
new file mode 100644
index 0000000..3417773
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/BooleanValueReader.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.value;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.decoder.ParquetRunLengthBitPackingHybridDecoder;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public final class BooleanValueReader extends AbstractValueReader {
+ private final ParquetRunLengthBitPackingHybridDecoder booleanReader;
+ private boolean nextValue;
+
+ public BooleanValueReader() {
+ booleanReader = new ParquetRunLengthBitPackingHybridDecoder(1);
+ }
+
+ @Override
+ public void resetValue(AbstractBytesInputStream in) {
+ booleanReader.reset(in);
+ }
+
+ @Override
+ public void nextValue() throws HyracksDataException {
+ try {
+ nextValue = booleanReader.readInt() == 1;
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+
+ }
+
+ @Override
+ public boolean getBoolean() {
+ return nextValue;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.BOOLEAN;
+ }
+
+ @Override
+ public int compareTo(AbstractValueReader o) {
+ return Boolean.compare(nextValue, o.getBoolean());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/DoubleValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/DoubleValueReader.java
new file mode 100644
index 0000000..24155f2
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/DoubleValueReader.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.value;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.decoder.ParquetDoublePlainValuesReader;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.om.types.ATypeTag;
+
+public final class DoubleValueReader extends AbstractValueReader {
+ private final ParquetDoublePlainValuesReader doubleReader;
+ private double nextValue;
+
+ public DoubleValueReader() {
+ doubleReader = new ParquetDoublePlainValuesReader();
+ }
+
+ @Override
+ public void resetValue(AbstractBytesInputStream in) throws IOException {
+ doubleReader.initFromPage(in);
+ }
+
+ @Override
+ public void nextValue() {
+ nextValue = doubleReader.readDouble();
+ }
+
+ @Override
+ public double getDouble() {
+ return nextValue;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.DOUBLE;
+ }
+
+ @Override
+ public int compareTo(AbstractValueReader o) {
+ return Double.compare(nextValue, o.getDouble());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/LongValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/LongValueReader.java
new file mode 100644
index 0000000..09413d9
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/LongValueReader.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.value;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.decoder.ParquetDeltaBinaryPackingValuesReader;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.om.types.ATypeTag;
+
+public final class LongValueReader extends AbstractValueReader {
+ private final ParquetDeltaBinaryPackingValuesReader longReader;
+ private long nextValue;
+
+ public LongValueReader() {
+ longReader = new ParquetDeltaBinaryPackingValuesReader();
+ }
+
+ @Override
+ public void resetValue(AbstractBytesInputStream in) throws IOException {
+ longReader.initFromPage(in);
+ }
+
+ @Override
+ public void nextValue() {
+ nextValue = longReader.readLong();
+ }
+
+ @Override
+ public long getLong() {
+ return nextValue;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.BIGINT;
+ }
+
+ @Override
+ public int compareTo(AbstractValueReader o) {
+ return Long.compare(nextValue, o.getLong());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java
new file mode 100644
index 0000000..fd56ff2
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.value;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class NoOpValueReader extends AbstractValueReader {
+ public static final AbstractValueReader INSTANCE = new NoOpValueReader();
+
+ private NoOpValueReader() {
+ }
+
+ @Override
+ public void resetValue(AbstractBytesInputStream in) throws IOException {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ @Override
+ public void nextValue() throws HyracksDataException {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ @Override
+ public int compareTo(AbstractValueReader o) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/StringValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/StringValueReader.java
new file mode 100644
index 0000000..8fd8874
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/StringValueReader.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.value;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.decoder.ParquetDeltaByteArrayReader;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+
+public final class StringValueReader extends AbstractValueReader {
+ private final ParquetDeltaByteArrayReader stringReader;
+ private IValueReference nextValue;
+
+ public StringValueReader() {
+ stringReader = new ParquetDeltaByteArrayReader(true);
+ }
+
+ @Override
+ public void resetValue(AbstractBytesInputStream in) throws IOException {
+ stringReader.initFromPage(in);
+ }
+
+ @Override
+ public void nextValue() {
+ nextValue = stringReader.readBytes();
+ }
+
+ @Override
+ public IValueReference getBytes() {
+ return nextValue;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.STRING;
+ }
+
+ @Override
+ public int compareTo(AbstractValueReader o) {
+ return UTF8StringPointable.compare(nextValue, o.getBytes());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/UUIDValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/UUIDValueReader.java
new file mode 100644
index 0000000..4f240e9
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/UUIDValueReader.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.reader.value;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.decoder.ParquetDeltaByteArrayReader;
+import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
+import org.apache.asterix.dataflow.data.nontagged.comparators.AUUIDPartialBinaryComparatorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public final class UUIDValueReader extends AbstractValueReader {
+ private final ParquetDeltaByteArrayReader uuidReader;
+ private IValueReference nextValue;
+
+ public UUIDValueReader() {
+ uuidReader = new ParquetDeltaByteArrayReader(false);
+ }
+
+ @Override
+ public void resetValue(AbstractBytesInputStream in) throws IOException {
+ uuidReader.initFromPage(in);
+ }
+
+ @Override
+ public void nextValue() {
+ nextValue = uuidReader.readBytes();
+ }
+
+ @Override
+ public IValueReference getBytes() {
+ return nextValue;
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return ATypeTag.UUID;
+ }
+
+ @Override
+ public int compareTo(AbstractValueReader o) {
+ return AUUIDPartialBinaryComparatorFactory.compare(nextValue, o.getBytes());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java
new file mode 100644
index 0000000..87eda82
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.asterix.column.bytes.encoder.ParquetRunLengthBitPackingHybridEncoder;
+import org.apache.asterix.column.util.ColumnValuesUtil;
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.IColumnValuesWriterFactory;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.values.writer.filters.NoOpColumnFilterWriter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+
+public abstract class AbstractColumnValuesWriter implements IColumnValuesWriter {
+ protected final AbstractColumnFilterWriter filterWriter;
+ protected final ParquetRunLengthBitPackingHybridEncoder definitionLevels;
+ protected final int level;
+
+ private final int columnIndex;
+ private final boolean collection;
+ private final int nullBitMask;
+ private int count;
+ private boolean writeValues;
+
+ AbstractColumnValuesWriter(int columnIndex, int level, boolean collection, boolean filtered) {
+ this.columnIndex = columnIndex;
+ this.level = level;
+ this.collection = collection;
+ nullBitMask = ColumnValuesUtil.getNullMask(level);
+ int width = ColumnValuesUtil.getBitWidth(level);
+ definitionLevels = new ParquetRunLengthBitPackingHybridEncoder(width);
+ this.filterWriter = filtered ? createFilter() : NoOpColumnFilterWriter.INSTANCE;
+ }
+
+ @Override
+ public final int getColumnIndex() {
+ return columnIndex;
+ }
+
+ @Override
+ public final int getEstimatedSize() {
+ return definitionLevels.getEstimatedSize() + getValuesEstimatedSize();
+ }
+
+ @Override
+ public final int getAllocatedSpace() {
+ return definitionLevels.getAllocatedSize() + getValuesAllocatedSize();
+ }
+
+ @Override
+ public final int getCount() {
+ return count;
+ }
+
+ @Override
+ public final void writeValue(ATypeTag tag, IValueReference value) throws HyracksDataException {
+ addLevel(level);
+ try {
+ addValue(tag, value);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public final void writeLevel(int level) throws HyracksDataException {
+ addLevel(level);
+ }
+
+ @Override
+ public void writeLevels(int level, int count) throws HyracksDataException {
+ writeValues = writeValues || this.level == level;
+ this.count += count;
+ try {
+ for (int i = 0; i < count; i++) {
+ definitionLevels.writeInt(level);
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public RunLengthIntArray getDefinitionLevelsIntArray() {
+ return null;
+ }
+
+ @Override
+ public final void writeNull(int level) throws HyracksDataException {
+ addLevel(level | nullBitMask);
+ }
+
+ @Override
+ public void writeValue(IColumnValuesReader reader) throws HyracksDataException {
+ try {
+ addValue(reader);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public void writeAntiMatter(ATypeTag tag, IValueReference value) throws HyracksDataException {
+ addLevel(0);
+ try {
+ addValue(tag, value);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public final void close() {
+ definitionLevels.close();
+ closeValues();
+ }
+
+ @Override
+ public final long getNormalizedMinValue() {
+ if (!writeValues) {
+ // ignore values as everything is missing/null
+ return Long.MAX_VALUE;
+ }
+ return filterWriter.getMinNormalizedValue();
+ }
+
+ @Override
+ public final long getNormalizedMaxValue() {
+ if (!writeValues) {
+ // ignore values as everything is missing/null
+ return Long.MIN_VALUE;
+ }
+ return filterWriter.getMaxNormalizedValue();
+ }
+
+ @Override
+ public final void flush(OutputStream out) throws HyracksDataException {
+ BytesInput values;
+ BytesInput defLevelBytes;
+ try {
+ BytesUtils.writeZigZagVarInt(level, out);
+ defLevelBytes = definitionLevels.toBytes();
+ BytesUtils.writeZigZagVarInt((int) defLevelBytes.size(), out);
+ BytesUtils.writeZigZagVarInt(count, out);
+ defLevelBytes.writeAllTo(out);
+ if (writeValues || collection) {
+ values = getBytes();
+ int valueSize = (int) values.size();
+ BytesUtils.writeZigZagVarInt(valueSize, out);
+ values.writeAllTo(out);
+ } else {
+ /*
+ * Do not write the values if all values are null/missing
+ */
+ BytesUtils.writeZigZagVarInt(0, out);
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ reset();
+ }
+
+ @Override
+ public final void reset() throws HyracksDataException {
+ definitionLevels.reset();
+ writeValues = false;
+ count = 0;
+ filterWriter.reset();
+ resetValues();
+ }
+
+ @Override
+ public final void serialize(DataOutput output) throws IOException {
+ output.write(getTypeTag().serialize());
+ output.writeInt(columnIndex);
+ output.writeInt(level);
+ output.writeBoolean(collection);
+ output.writeBoolean(filterWriter != NoOpColumnFilterWriter.INSTANCE);
+ }
+
+ public static IColumnValuesWriter deserialize(DataInput input, IColumnValuesWriterFactory writerFactory)
+ throws IOException {
+ ATypeTag typeTag = ATypeTag.VALUE_TYPE_MAPPING[input.readByte()];
+ int columnIndex = input.readInt();
+ int level = input.readInt();
+ boolean collection = input.readBoolean();
+ boolean filtered = input.readBoolean();
+ return writerFactory.createValueWriter(typeTag, columnIndex, level, collection, filtered);
+ }
+
+ protected void addLevel(int level) throws HyracksDataException {
+ try {
+ writeValues = writeValues || this.level == level;
+ definitionLevels.writeInt(level);
+ count++;
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ protected abstract ATypeTag getTypeTag();
+
+ protected abstract void addValue(ATypeTag tag, IValueReference value) throws IOException;
+
+ protected abstract void addValue(IColumnValuesReader reader) throws IOException;
+
+ protected abstract BytesInput getBytes() throws IOException;
+
+ protected abstract int getValuesEstimatedSize();
+
+ protected abstract int getValuesAllocatedSize();
+
+ protected abstract AbstractColumnFilterWriter createFilter();
+
+ protected abstract void resetValues() throws HyracksDataException;
+
+ protected abstract void closeValues();
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java
new file mode 100644
index 0000000..0058f18
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.encoder.ParquetRunLengthBitPackingHybridEncoder;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.values.writer.filters.LongColumnFilterWriter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.bytes.BytesInput;
+
+public final class BooleanColumnValuesWriter extends AbstractColumnValuesWriter {
+ private final ParquetRunLengthBitPackingHybridEncoder booleanWriter;
+
+ public BooleanColumnValuesWriter(int columnIndex, int level, boolean collection, boolean filtered) {
+ super(columnIndex, level, collection, filtered);
+ booleanWriter = new ParquetRunLengthBitPackingHybridEncoder(1);
+ }
+
+ @Override
+ protected void addValue(ATypeTag tag, IValueReference value) throws IOException {
+ byte booleanValue = value.getByteArray()[value.getStartOffset()];
+ booleanWriter.writeInt(booleanValue);
+ filterWriter.addLong(booleanValue);
+ }
+
+ @Override
+ protected void resetValues() {
+ booleanWriter.reset();
+ }
+
+ @Override
+ protected BytesInput getBytes() throws IOException {
+ return booleanWriter.toBytes();
+ }
+
+ @Override
+ protected int getValuesEstimatedSize() {
+ return booleanWriter.getEstimatedSize();
+ }
+
+ @Override
+ protected int getValuesAllocatedSize() {
+ return booleanWriter.getAllocatedSize();
+ }
+
+ @Override
+ protected void addValue(IColumnValuesReader reader) throws IOException {
+ int value = reader.getBoolean() ? 1 : 0;
+ booleanWriter.writeInt(value);
+ }
+
+ @Override
+ protected AbstractColumnFilterWriter createFilter() {
+ return new LongColumnFilterWriter();
+ }
+
+ @Override
+ protected void closeValues() {
+ booleanWriter.close();
+ }
+
+ @Override
+ protected ATypeTag getTypeTag() {
+ return ATypeTag.BOOLEAN;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnBatchWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnBatchWriter.java
new file mode 100644
index 0000000..490afe7
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnBatchWriter.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import static org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter.FILTER_SIZE;
+
+import java.nio.ByteBuffer;
+import java.util.PriorityQueue;
+
+import org.apache.asterix.column.bytes.stream.out.ByteBufferOutputStream;
+import org.apache.asterix.column.bytes.stream.out.MultiPersistentBufferBytesOutputStream;
+import org.apache.asterix.column.bytes.stream.out.pointer.IReservedPointer;
+import org.apache.asterix.column.values.IColumnBatchWriter;
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+/**
+ * A writer for a batch columns' values
+ */
+public final class ColumnBatchWriter implements IColumnBatchWriter {
+ private final ByteBufferOutputStream primaryKeys;
+ private final MultiPersistentBufferBytesOutputStream columns;
+ private final int pageSize;
+ private final float tolerance;
+ private final IReservedPointer columnLengthPointer;
+
+ private ByteBuffer pageZero;
+ private int columnsOffset;
+ private int filtersOffset;
+ private int primaryKeysOffset;
+ private int nonKeyColumnStartOffset;
+
+ public ColumnBatchWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef, int pageSize, float tolerance) {
+ this.pageSize = pageSize;
+ this.tolerance = tolerance;
+ primaryKeys = new ByteBufferOutputStream();
+ columns = new MultiPersistentBufferBytesOutputStream(multiPageOpRef);
+ columnLengthPointer = columns.createPointer();
+ }
+
+ @Override
+ public void setPageZeroBuffer(ByteBuffer pageZero, int numberOfColumns, int numberOfPrimaryKeys) {
+ this.pageZero = pageZero;
+ int offset = pageZero.position();
+
+ columnsOffset = offset;
+ offset += numberOfColumns * Integer.BYTES;
+
+ filtersOffset = offset;
+ offset += numberOfColumns * FILTER_SIZE;
+
+ pageZero.position(offset);
+ primaryKeysOffset = offset;
+ primaryKeys.reset(pageZero);
+ nonKeyColumnStartOffset = pageZero.capacity();
+ }
+
+ @Override
+ public int writePrimaryKeyColumns(IColumnValuesWriter[] primaryKeyWriters) throws HyracksDataException {
+ int allocatedSpace = 0;
+ for (int i = 0; i < primaryKeyWriters.length; i++) {
+ IColumnValuesWriter writer = primaryKeyWriters[i];
+ setColumnOffset(i, primaryKeysOffset + primaryKeys.size());
+ writer.flush(primaryKeys);
+ allocatedSpace += writer.getAllocatedSpace();
+ }
+ return allocatedSpace;
+ }
+
+ @Override
+ public int writeColumns(PriorityQueue<IColumnValuesWriter> nonKeysColumnWriters) throws HyracksDataException {
+ int allocatedSpace = 0;
+ columns.reset();
+ while (!nonKeysColumnWriters.isEmpty()) {
+ IColumnValuesWriter writer = nonKeysColumnWriters.poll();
+ writeColumn(writer);
+ allocatedSpace += writer.getAllocatedSpace();
+ }
+ return allocatedSpace;
+ }
+
+ private void writeColumn(IColumnValuesWriter writer) throws HyracksDataException {
+ if (!hasEnoughSpace(columns.getCurrentBufferPosition(), writer)) {
+ /*
+ * We reset the columns stream to write all pages and confiscate a new buffer to minimize splitting
+ * the columns value into multiple pages.
+ */
+ nonKeyColumnStartOffset += columns.capacity();
+ columns.reset();
+ }
+
+ int columnRelativeOffset = columns.size();
+ columns.reserveInteger(columnLengthPointer);
+ setColumnOffset(writer.getColumnIndex(), nonKeyColumnStartOffset + columnRelativeOffset);
+
+ writeFilter(writer);
+ writer.flush(columns);
+
+ int length = columns.size() - columnRelativeOffset;
+ columnLengthPointer.setInteger(length);
+ }
+
+ private boolean hasEnoughSpace(int bufferPosition, IColumnValuesWriter columnWriter) {
+ //Estimated size mostly overestimate the size
+ int columnSize = columnWriter.getEstimatedSize();
+ float remainingPercentage = (pageSize - bufferPosition) / (float) pageSize;
+ if (columnSize > pageSize) {
+ /*
+ * If the column size is larger than the page size, we check whether the remaining space is less than
+ * the tolerance percentage
+ * - true --> allocate new buffer and tolerate empty space
+ * - false --> we split the column into two pages
+ */
+ return remainingPercentage >= tolerance;
+ }
+
+ int freeSpace = pageSize - (bufferPosition + columnSize);
+
+ /*
+ * Check if the free space is enough to fit the column or the free space is less that the tolerance percentage
+ * - true --> we allocate new buffer and tolerate empty space
+ * - false --> we split the column into two pages
+ */
+ return freeSpace > columnSize || remainingPercentage >= tolerance;
+ }
+
+ private void setColumnOffset(int columnIndex, int offset) {
+ pageZero.putInt(columnsOffset + Integer.BYTES * columnIndex, offset);
+ }
+
+ private void writeFilter(IColumnValuesWriter writer) {
+ int offset = filtersOffset + writer.getColumnIndex() * FILTER_SIZE;
+ pageZero.putLong(offset, writer.getNormalizedMinValue());
+ pageZero.putLong(offset + Long.BYTES, writer.getNormalizedMaxValue());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnValuesWriterFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnValuesWriterFactory.java
new file mode 100644
index 0000000..6a514ff
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnValuesWriterFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import org.apache.asterix.column.values.IColumnValuesWriter;
+import org.apache.asterix.column.values.IColumnValuesWriterFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+public class ColumnValuesWriterFactory implements IColumnValuesWriterFactory {
+ private final Mutable<IColumnWriteMultiPageOp> multiPageOpRef;
+
+ public ColumnValuesWriterFactory(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ this.multiPageOpRef = multiPageOpRef;
+ }
+
+ @Override
+ public IColumnValuesWriter createValueWriter(ATypeTag typeTag, int columnIndex, int maxLevel, boolean writeAlways,
+ boolean filtered) {
+ switch (typeTag) {
+ case MISSING:
+ case NULL:
+ return new NullMissingColumnValuesWriter(columnIndex, maxLevel, writeAlways, filtered);
+ case BOOLEAN:
+ return new BooleanColumnValuesWriter(columnIndex, maxLevel, writeAlways, filtered);
+ case BIGINT:
+ return new LongColumnValuesWriter(multiPageOpRef, columnIndex, maxLevel, writeAlways, filtered);
+ case DOUBLE:
+ return new DoubleColumnValuesWriter(multiPageOpRef, columnIndex, maxLevel, writeAlways, filtered);
+ case STRING:
+ return new StringColumnValuesWriter(multiPageOpRef, columnIndex, maxLevel, writeAlways, filtered);
+ case UUID:
+ return new UUIDColumnValuesWriter(multiPageOpRef, columnIndex, maxLevel, writeAlways, filtered);
+ default:
+ throw new UnsupportedOperationException(typeTag + " is not supported");
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java
new file mode 100644
index 0000000..ca5cbb1
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.encoder.ParquetPlainValuesWriter;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.values.writer.filters.DoubleColumnFilterWriter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+import org.apache.hyracks.data.std.primitive.FloatPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.data.std.primitive.ShortPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.bytes.BytesInput;
+
+public final class DoubleColumnValuesWriter extends AbstractColumnValuesWriter {
+ private final ParquetPlainValuesWriter doubleWriter;
+
+ public DoubleColumnValuesWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef, int columnIndex, int level,
+ boolean collection, boolean filtered) {
+ super(columnIndex, level, collection, filtered);
+ doubleWriter = new ParquetPlainValuesWriter(multiPageOpRef);
+ }
+
+ @Override
+ protected void addValue(ATypeTag tag, IValueReference value) throws IOException {
+ final double normalizedDouble = getValue(tag, value.getByteArray(), value.getStartOffset());
+ doubleWriter.writeDouble(normalizedDouble);
+ filterWriter.addDouble(normalizedDouble);
+ }
+
+ private double getValue(ATypeTag typeTag, byte[] byteArray, int offset) {
+ switch (typeTag) {
+ case TINYINT:
+ return byteArray[offset];
+ case SMALLINT:
+ return ShortPointable.getShort(byteArray, offset);
+ case INTEGER:
+ return IntegerPointable.getInteger(byteArray, offset);
+ case BIGINT:
+ return LongPointable.getLong(byteArray, offset);
+ case FLOAT:
+ return FloatPointable.getFloat(byteArray, offset);
+ case DOUBLE:
+ return DoublePointable.getDouble(byteArray, offset);
+ default:
+ throw new IllegalAccessError(typeTag + "is not of floating type");
+ }
+ }
+
+ @Override
+ protected void resetValues() throws HyracksDataException {
+ doubleWriter.reset();
+ }
+
+ @Override
+ protected BytesInput getBytes() throws IOException {
+ return doubleWriter.getBytes();
+ }
+
+ @Override
+ protected int getValuesEstimatedSize() {
+ return doubleWriter.getEstimatedSize();
+ }
+
+ @Override
+ protected int getValuesAllocatedSize() {
+ return doubleWriter.getAllocatedSize();
+ }
+
+ @Override
+ protected void addValue(IColumnValuesReader reader) throws IOException {
+ double value = reader.getDouble();
+ doubleWriter.writeDouble(value);
+ filterWriter.addDouble(value);
+ }
+
+ @Override
+ protected AbstractColumnFilterWriter createFilter() {
+ return new DoubleColumnFilterWriter();
+ }
+
+ @Override
+ protected void closeValues() {
+ doubleWriter.close();
+ }
+
+ @Override
+ protected ATypeTag getTypeTag() {
+ return ATypeTag.DOUBLE;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java
new file mode 100644
index 0000000..e71ec73
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.encoder.ParquetDeltaBinaryPackingValuesWriterForLong;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.values.writer.filters.LongColumnFilterWriter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.data.std.primitive.ShortPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.bytes.BytesInput;
+
+final class LongColumnValuesWriter extends AbstractColumnValuesWriter {
+ private final ParquetDeltaBinaryPackingValuesWriterForLong longWriter;
+
+ public LongColumnValuesWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef, int columnIndex, int level,
+ boolean collection, boolean filtered) {
+ super(columnIndex, level, collection, filtered);
+ longWriter = new ParquetDeltaBinaryPackingValuesWriterForLong(multiPageOpRef);
+ }
+
+ @Override
+ protected void addValue(ATypeTag tag, IValueReference value) throws IOException {
+ final long normalizedInt = getValue(tag, value.getByteArray(), value.getStartOffset());
+ longWriter.writeLong(normalizedInt);
+ filterWriter.addLong(normalizedInt);
+ }
+
+ private long getValue(ATypeTag typeTag, byte[] byteArray, int offset) {
+ switch (typeTag) {
+ case TINYINT:
+ return byteArray[offset];
+ case SMALLINT:
+ return ShortPointable.getShort(byteArray, offset);
+ case INTEGER:
+ return IntegerPointable.getInteger(byteArray, offset);
+ case BIGINT:
+ return LongPointable.getLong(byteArray, offset);
+ default:
+ throw new IllegalAccessError(typeTag + "is not of type integer");
+ }
+ }
+
+ @Override
+ protected void resetValues() throws HyracksDataException {
+ longWriter.reset();
+ }
+
+ @Override
+ protected BytesInput getBytes() throws IOException {
+ return longWriter.getBytes();
+ }
+
+ @Override
+ protected int getValuesEstimatedSize() {
+ return longWriter.getEstimatedSize();
+ }
+
+ @Override
+ protected int getValuesAllocatedSize() {
+ return longWriter.getAllocatedSize();
+ }
+
+ @Override
+ protected void addValue(IColumnValuesReader reader) throws IOException {
+ long value = reader.getLong();
+ longWriter.writeLong(value);
+ filterWriter.addLong(value);
+ }
+
+ @Override
+ protected AbstractColumnFilterWriter createFilter() {
+ return new LongColumnFilterWriter();
+ }
+
+ @Override
+ protected void closeValues() {
+ longWriter.close();
+ }
+
+ @Override
+ protected ATypeTag getTypeTag() {
+ return ATypeTag.BIGINT;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java
new file mode 100644
index 0000000..edc9fe2
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.util.RunLengthIntArray;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.values.writer.filters.NoOpColumnFilterWriter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.bytes.BytesInput;
+
+public class NullMissingColumnValuesWriter extends AbstractColumnValuesWriter {
+ private static final BytesInput EMPTY = BytesInput.empty();
+ private final RunLengthIntArray defLevelsIntArray;
+
+ NullMissingColumnValuesWriter(int columnIndex, int level, boolean collection, boolean filtered) {
+ super(columnIndex, level, collection, filtered);
+ defLevelsIntArray = new RunLengthIntArray();
+ }
+
+ @Override
+ protected void addLevel(int level) throws HyracksDataException {
+ defLevelsIntArray.add(level);
+ super.addLevel(level);
+ }
+
+ @Override
+ public void writeLevels(int level, int count) throws HyracksDataException {
+ defLevelsIntArray.add(level, count);
+ super.writeLevels(level, count);
+ }
+
+ @Override
+ protected ATypeTag getTypeTag() {
+ return ATypeTag.NULL;
+ }
+
+ @Override
+ protected void addValue(ATypeTag tag, IValueReference value) throws IOException {
+ throw new IllegalStateException("Null writer should not add value");
+ }
+
+ @Override
+ protected void addValue(IColumnValuesReader reader) throws IOException {
+ throw new IllegalStateException("Null writer should not add value");
+ }
+
+ @Override
+ protected BytesInput getBytes() throws IOException {
+ return EMPTY;
+ }
+
+ @Override
+ protected int getValuesEstimatedSize() {
+ return 0;
+ }
+
+ @Override
+ protected int getValuesAllocatedSize() {
+ return 0;
+ }
+
+ @Override
+ protected AbstractColumnFilterWriter createFilter() {
+ return NoOpColumnFilterWriter.INSTANCE;
+ }
+
+ @Override
+ protected void resetValues() throws HyracksDataException {
+ defLevelsIntArray.reset();
+ }
+
+ @Override
+ protected void closeValues() {
+ defLevelsIntArray.reset();
+ }
+
+ @Override
+ public RunLengthIntArray getDefinitionLevelsIntArray() {
+ return defLevelsIntArray;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java
new file mode 100644
index 0000000..e1a3ffd
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import java.io.IOException;
+
+import org.apache.asterix.column.bytes.encoder.ParquetDeltaByteArrayWriter;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.values.writer.filters.StringColumnFilterWriter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.parquet.bytes.BytesInput;
+
+public class StringColumnValuesWriter extends AbstractColumnValuesWriter {
+ private final ParquetDeltaByteArrayWriter stringWriter;
+ private final boolean skipLengthBytes;
+
+ public StringColumnValuesWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef, int columnIndex, int level,
+ boolean collection, boolean filtered) {
+ this(multiPageOpRef, columnIndex, level, collection, filtered, true);
+ }
+
+ protected StringColumnValuesWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef, int columnIndex, int level,
+ boolean collection, boolean filtered, boolean skipLengthBytes) {
+ super(columnIndex, level, collection, filtered);
+ stringWriter = new ParquetDeltaByteArrayWriter(multiPageOpRef);
+ this.skipLengthBytes = skipLengthBytes;
+ }
+
+ @Override
+ protected final void addValue(ATypeTag tag, IValueReference value) throws IOException {
+ stringWriter.writeBytes(value, skipLengthBytes);
+ filterWriter.addValue(value);
+ }
+
+ @Override
+ protected final void resetValues() throws HyracksDataException {
+ stringWriter.reset();
+ }
+
+ @Override
+ protected final BytesInput getBytes() throws IOException {
+ return stringWriter.getBytes();
+ }
+
+ @Override
+ protected final int getValuesEstimatedSize() {
+ return stringWriter.getEstimatedSize();
+ }
+
+ @Override
+ protected final int getValuesAllocatedSize() {
+ return stringWriter.getAllocatedSize();
+ }
+
+ @Override
+ protected final void addValue(IColumnValuesReader reader) throws IOException {
+ IValueReference value = reader.getBytes();
+ stringWriter.writeBytes(value, skipLengthBytes);
+ filterWriter.addValue(value);
+ }
+
+ @Override
+ protected AbstractColumnFilterWriter createFilter() {
+ return new StringColumnFilterWriter();
+ }
+
+ @Override
+ protected final void closeValues() {
+ stringWriter.close();
+ }
+
+ @Override
+ protected ATypeTag getTypeTag() {
+ return ATypeTag.STRING;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/UUIDColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/UUIDColumnValuesWriter.java
new file mode 100644
index 0000000..1e98754
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/UUIDColumnValuesWriter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer;
+
+import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.values.writer.filters.UUIDColumnFilterWriter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+final class UUIDColumnValuesWriter extends StringColumnValuesWriter {
+
+ public UUIDColumnValuesWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef, int columnIndex, int level,
+ boolean collection, boolean filtered) {
+ super(multiPageOpRef, columnIndex, level, collection, filtered, false);
+ }
+
+ @Override
+ protected AbstractColumnFilterWriter createFilter() {
+ return new UUIDColumnFilterWriter();
+ }
+
+ @Override
+ protected ATypeTag getTypeTag() {
+ return ATypeTag.UUID;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/AbstractColumnFilterWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/AbstractColumnFilterWriter.java
new file mode 100644
index 0000000..abbe314
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/AbstractColumnFilterWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer.filters;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public abstract class AbstractColumnFilterWriter {
+ public static final int FILTER_SIZE = Long.BYTES * 2;
+
+ public void addLong(long value) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public void addDouble(double value) {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public void addValue(IValueReference value) throws HyracksDataException {
+ throw new UnsupportedOperationException(getClass().getName());
+ }
+
+ public abstract long getMinNormalizedValue();
+
+ public abstract long getMaxNormalizedValue();
+
+ public abstract void reset();
+
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/DoubleColumnFilterWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/DoubleColumnFilterWriter.java
new file mode 100644
index 0000000..6fccabe
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/DoubleColumnFilterWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer.filters;
+
+public class DoubleColumnFilterWriter extends AbstractColumnFilterWriter {
+ private double min;
+ private double max;
+
+ public DoubleColumnFilterWriter() {
+ reset();
+ }
+
+ @Override
+ public void addDouble(double value) {
+ min = Math.min(min, value);
+ max = Math.max(max, value);
+ }
+
+ @Override
+ public long getMinNormalizedValue() {
+ return Double.doubleToLongBits(min);
+ }
+
+ @Override
+ public long getMaxNormalizedValue() {
+ return Double.doubleToLongBits(max);
+ }
+
+ @Override
+ public void reset() {
+ min = Double.MIN_VALUE;
+ max = Double.MAX_VALUE;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/LongColumnFilterWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/LongColumnFilterWriter.java
new file mode 100644
index 0000000..25cb94c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/LongColumnFilterWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer.filters;
+
+public class LongColumnFilterWriter extends AbstractColumnFilterWriter {
+ private long min;
+ private long max;
+
+ public LongColumnFilterWriter() {
+ reset();
+ }
+
+ @Override
+ public void addLong(long value) {
+ min = Math.min(min, value);
+ max = Math.max(max, value);
+ }
+
+ @Override
+ public long getMinNormalizedValue() {
+ return min;
+ }
+
+ @Override
+ public long getMaxNormalizedValue() {
+ return max;
+ }
+
+ @Override
+ public void reset() {
+ min = Long.MAX_VALUE;
+ max = Long.MIN_VALUE;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/NoOpColumnFilterWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/NoOpColumnFilterWriter.java
new file mode 100644
index 0000000..c4f6f6f
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/NoOpColumnFilterWriter.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer.filters;
+
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public class NoOpColumnFilterWriter extends AbstractColumnFilterWriter {
+ public static final AbstractColumnFilterWriter INSTANCE = new NoOpColumnFilterWriter();
+
+ private NoOpColumnFilterWriter() {
+ }
+
+ @Override
+ public void addLong(long value) {
+ //NoOp
+ }
+
+ @Override
+ public void addDouble(double value) {
+ //NoOp
+ }
+
+ @Override
+ public void addValue(IValueReference value) {
+ //NoOp
+ }
+
+ @Override
+ public long getMinNormalizedValue() {
+ return 0;
+ }
+
+ @Override
+ public long getMaxNormalizedValue() {
+ return 0;
+ }
+
+ @Override
+ public void reset() {
+ //NoOp
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/StringColumnFilterWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/StringColumnFilterWriter.java
new file mode 100644
index 0000000..77d82c0
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/StringColumnFilterWriter.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer.filters;
+
+import static org.apache.hyracks.util.string.UTF8StringUtil.charAt;
+import static org.apache.hyracks.util.string.UTF8StringUtil.charSize;
+import static org.apache.hyracks.util.string.UTF8StringUtil.getNumBytesToStoreLength;
+import static org.apache.hyracks.util.string.UTF8StringUtil.getUTFLength;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public class StringColumnFilterWriter extends LongColumnFilterWriter {
+ @Override
+ public void addValue(IValueReference value) throws HyracksDataException {
+ addLong(normalize(value));
+ }
+
+ /**
+ * Normalizes the string in a {@link Long}
+ *
+ * @see org.apache.hyracks.util.string.UTF8StringUtil#normalize(byte[], int)
+ */
+ public static long normalize(IValueReference value) {
+ byte[] bytes = value.getByteArray();
+ int start = value.getStartOffset();
+
+ long nk = 0;
+ int offset = start + getNumBytesToStoreLength(getUTFLength(bytes, start));
+ int end = start + value.getLength();
+ for (int i = 0; i < 4; ++i) {
+ nk <<= 16;
+ if (offset < end) {
+ nk += (charAt(bytes, offset)) & 0xffff;
+ offset += charSize(bytes, offset);
+ }
+ }
+ return nk >>> 1;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/UUIDColumnFilterWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/UUIDColumnFilterWriter.java
new file mode 100644
index 0000000..5e2bc61
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/UUIDColumnFilterWriter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.values.writer.filters;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+
+/**
+ * UUID filters are the LSB component of all written UUIDs. This could provide false positives UUIDs; however, this
+ * still can filter out non-matching UUIDs.
+ */
+public class UUIDColumnFilterWriter extends LongColumnFilterWriter {
+
+ @Override
+ public void addValue(IValueReference value) throws HyracksDataException {
+ addLong(getLSB(value));
+ }
+
+ public static long getLSB(IValueReference value) {
+ byte[] bytes = value.getByteArray();
+ int start = value.getStartOffset();
+ return LongPointable.getLong(bytes, start + Long.BYTES);
+ }
+}
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index e76fb5b..cd20488 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-common</artifactId>
<licenses>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 325c238..5466a3b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -105,6 +105,10 @@
BOOLEAN,
AlgebricksConfig.ARRAY_INDEX_DEFAULT,
"Enable/disable using array-indexes in queries"),
+ COMPILER_BATCH_LOOKUP(
+ BOOLEAN,
+ AlgebricksConfig.BATCH_LOOKUP_DEFAULT,
+ "Enable/disable batch point-lookups when running queries with secondary indexes"),
COMPILER_CBO(BOOLEAN, AlgebricksConfig.CBO_DEFAULT, "Set the mode for cost based optimization"),
COMPILER_CBOTEST(BOOLEAN, AlgebricksConfig.CBO_TEST_DEFAULT, "Set the mode for cost based optimization"),
COMPILER_FORCEJOINORDER(
@@ -114,7 +118,11 @@
COMPILER_QUERYPLANSHAPE(
STRING,
AlgebricksConfig.QUERY_PLAN_SHAPE_DEFAULT,
- "Set the mode for forcing the shape of the query plan");
+ "Set the mode for forcing the shape of the query plan"),
+ COMPILER_COLUMN_FILTER(
+ BOOLEAN,
+ AlgebricksConfig.COLUMN_FILTER_DEFAULT,
+ "Enable/disable the use of column min/max filters");
private final IOptionType type;
private final Object defaultValue;
@@ -184,6 +192,8 @@
public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
+ public static final String COMPILER_BATCHED_LOOKUP_KEY = Option.COMPILER_BATCH_LOOKUP.ini();
+
public static final String COMPILER_CBO_KEY = Option.COMPILER_CBO.ini();
public static final String COMPILER_CBO_TEST_KEY = Option.COMPILER_CBOTEST.ini();
@@ -192,6 +202,8 @@
public static final String COMPILER_QUERY_PLAN_SHAPE_KEY = Option.COMPILER_QUERYPLANSHAPE.ini();
+ public static final String COMPILER_COLUMN_FILTER_KEY = Option.COMPILER_COLUMN_FILTER.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@ -266,6 +278,10 @@
return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
}
+ public boolean isBatchLookup() {
+ return accessor.getBoolean(Option.COMPILER_BATCH_LOOKUP);
+ }
+
public boolean getCBOMode() {
return accessor.getBoolean(Option.COMPILER_CBO);
}
@@ -286,4 +302,8 @@
return AlgebricksConfig.QUERY_PLAN_SHAPE_DEFAULT;
return queryPlanShapeMode;
}
+
+ public boolean isColumnFilter() {
+ return accessor.getBoolean(Option.COMPILER_COLUMN_FILTER);
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
index 478bd46..3f15eef 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.common.config;
+import java.util.HashMap;
+import java.util.Map;
+
public class DatasetConfig {
/*
@@ -42,7 +45,7 @@
LENGTH_PARTITIONED_WORD_INVIX,
LENGTH_PARTITIONED_NGRAM_INVIX,
ARRAY,
- SAMPLE;
+ SAMPLE
}
public enum TransactionState {
@@ -79,4 +82,34 @@
*/
APPEND_OP
}
+
+ public enum DatasetFormat {
+ /**
+ * Row format using ADM
+ */
+ ROW("row"),
+ /**
+ * Column format using AMAX
+ */
+ COLUMN("column");
+
+ private final String format;
+ private static final Map<String, DatasetFormat> FORMATS = createFormats();
+
+ DatasetFormat(String format) {
+ this.format = format;
+ }
+
+ private static Map<String, DatasetFormat> createFormats() {
+ Map<String, DatasetFormat> formats = new HashMap<>();
+ for (DatasetFormat format : DatasetFormat.values()) {
+ formats.put(format.format, format);
+ }
+ return formats;
+ }
+
+ public static DatasetFormat getFormat(String format) {
+ return FORMATS.get(format.trim().toLowerCase());
+ }
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index ae70475..0e8ac55 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -77,6 +77,8 @@
int externalScanBufferSize = getExternalScanBufferSize(
(String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
compilerProperties.getExternalScanMemorySize(), sourceLoc);
+ boolean batchLookup = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_BATCHED_LOOKUP_KEY,
+ compilerProperties.isBatchLookup());
boolean cbo =
getBoolean(querySpecificConfig, CompilerProperties.COMPILER_CBO_KEY, compilerProperties.getCBOMode());
boolean cboTest = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_CBO_TEST_KEY,
@@ -85,6 +87,8 @@
compilerProperties.getForceJoinOrderMode());
String queryPlanShape = getString(querySpecificConfig, CompilerProperties.COMPILER_QUERY_PLAN_SHAPE_KEY,
compilerProperties.getQueryPlanShapeMode());
+ boolean columnFilter = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_COLUMN_FILTER_KEY,
+ compilerProperties.isColumnFilter());
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
@@ -103,10 +107,12 @@
physOptConf.setMinMemoryAllocation(minMemoryAllocation);
physOptConf.setArrayIndexEnabled(arrayIndex);
physOptConf.setExternalScanBufferSize(externalScanBufferSize);
+ physOptConf.setBatchLookup(batchLookup);
physOptConf.setCBOMode(cbo);
physOptConf.setCBOTestMode(cboTest);
physOptConf.setForceJoinOrderMode(forceJoinOrder);
physOptConf.setQueryPlanShapeMode(queryPlanShape);
+ physOptConf.setColumnFilter(columnFilter);
return physOptConf;
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index 12c9c68..5b99fa0 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -56,12 +56,14 @@
STORAGE_COMPRESSION_BLOCK(STRING, "snappy"),
STORAGE_DISK_FORCE_BYTES(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(16, MEGABYTE)),
STORAGE_IO_SCHEDULER(STRING, "greedy"),
- STORAGE_WRITE_RATE_LIMIT(LONG_BYTE_UNIT, 0l),
+ STORAGE_WRITE_RATE_LIMIT(LONG_BYTE_UNIT, 0L),
STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
STORAGE_MAX_SCHEDULED_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 8),
STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
STORAGE_GLOBAL_CLEANUP(BOOLEAN, true),
- STORAGE_GLOBAL_CLEANUP_TIMEOUT(POSITIVE_INTEGER, (int) TimeUnit.MINUTES.toSeconds(10));
+ STORAGE_GLOBAL_CLEANUP_TIMEOUT(POSITIVE_INTEGER, (int) TimeUnit.MINUTES.toSeconds(10)),
+ STORAGE_COLUMN_MAX_TUPLE_COUNT(NONNEGATIVE_INTEGER, 15000),
+ STORAGE_COLUMN_FREE_SPACE_TOLERANCE(DOUBLE, 0.15);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -129,6 +131,11 @@
return "Indicates whether or not global storage cleanup is performed";
case STORAGE_GLOBAL_CLEANUP_TIMEOUT:
return "The maximum time to wait for nodes to respond to global storage cleanup requests";
+ case STORAGE_COLUMN_MAX_TUPLE_COUNT:
+ return "The maximum number of tuples to be stored per a mega leaf page";
+ case STORAGE_COLUMN_FREE_SPACE_TOLERANCE:
+ return "The percentage of the maximum tolerable empty space for a physical mega leaf page (e.g.,"
+ + " 0.15 means a physical page with 15% or less empty space is tolerable)";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -265,4 +272,12 @@
public int getDiskForcePages() {
return (int) (accessor.getLong(Option.STORAGE_DISK_FORCE_BYTES) / getBufferCachePageSize());
}
+
+ public int getColumnMaxTupleCount() {
+ return accessor.getInt(Option.STORAGE_COLUMN_MAX_TUPLE_COUNT);
+ }
+
+ public float getColumnFreeSpaceTolerance() {
+ return (float) accessor.getDouble(Option.STORAGE_COLUMN_FREE_SPACE_TOLERANCE);
+ }
}
diff --git a/asterixdb/asterix-coverage/pom.xml b/asterixdb/asterix-coverage/pom.xml
index 19b73b9..d29f5a5 100644
--- a/asterixdb/asterix-coverage/pom.xml
+++ b/asterixdb/asterix-coverage/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/asterixdb/asterix-dashboard/pom.xml b/asterixdb/asterix-dashboard/pom.xml
index 9d962f8..0418876 100644
--- a/asterixdb/asterix-dashboard/pom.xml
+++ b/asterixdb/asterix-dashboard/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>asterix-dashboard</artifactId>
diff --git a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.scss b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.scss
index 94311f8..68c12f4 100755
--- a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.scss
+++ b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.scss
@@ -46,6 +46,7 @@
.options {
display: flex;
flex-flow: row;
+ overflow: auto;
}
.codemirror-container {
diff --git a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.ts b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.ts
index f885edb..cb46a6a 100755
--- a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.ts
+++ b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/input.component.ts
@@ -538,7 +538,7 @@
this.queryString = '';
this.selected = 'Default';
} else {
- this.queryString = 'USE ' + this.selected + '; \n';
+ this.queryString = 'USE ' + this.selected + '; \n' + this.queryString;
}
this.editor.getDoc().setValue(this.queryString);
this.editor.execCommand('goDocEnd')
diff --git a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.html b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.html
index a131353..4b6ecc5 100644
--- a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.html
+++ b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.html
@@ -97,7 +97,7 @@
[links]="edgesArr"
[nodes]="nodesArr"
[draggingEnabled]="false"
- [zoomSpeed]="0.025"
+ [zoomSpeed]="0.015"
[update$]="update$"
[layoutSettings]="{
orientation: planOrientation,
diff --git a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.scss b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.scss
index 6c40a68..9601675 100644
--- a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.scss
+++ b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/plan-viewer.component.scss
@@ -36,7 +36,6 @@
}
.panel {
- order: 2;
display: flex;
flex-flow: column;
justify-content: stretch;
diff --git a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.html b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.html
index 0fbf6f6..c5754762 100644
--- a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.html
+++ b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.html
@@ -30,7 +30,8 @@
<button *ngIf="isCSV" mat-button class='button export' (click)='openJSONExportPicker()' matTooltip="Export CSV file to Computer">EXPORT</button>
<button *ngIf="isCSV == false" mat-button class='button export' (click)='openJSONExportPicker()' matTooltip="Export JSON/JSONL file to Computer">EXPORT</button>
</span>
- <mat-paginator *ngIf="this.planVisible === false" [showFirstLastButtons]="true" [length]='metrics.resultCount' [pageSize]='pagedefaults.pageSize' [pageSizeOptions]='pageSizeOptions' (page)='showResults($event, false)'>
+
+ <mat-paginator *ngIf="this.planVisible === false" [showFirstLastButtons]="true" [length]='metrics.resultCount' [pageSize]='pageSize' [pageSizeOptions]='pageSizeOptions' (page)='showResults($event, false)'>
</mat-paginator>
</div>
<div *ngIf='treeVisible' class='navi-data' class='navi-data'>
diff --git a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.ts b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.ts
index 5309991..66a6dd6 100644
--- a/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.ts
+++ b/asterixdb/asterix-dashboard/src/node/src/app/dashboard/query/tree-view.component.ts
@@ -60,7 +60,8 @@
currentRange: any;
/* see 10 records as initial set */
pagedefaults: any = { pageIndex: 0, pageSize:10, lenght: 0};
- pageSizeOptions = [5, 10, 25, 100, 200];
+ pageSize = 10;
+ pageSizeOptions = [5, 10, 25, 100, 200, 300, 400];
viewMode = 'JSON';
showGoTop = false;
showGoBottom = false;
@@ -141,6 +142,9 @@
}
showResults(range, expanded) {
+ // update pagesize
+ this.pageSize = range.pageSize;
+
this.currentRange = range;
this.currentIndex = this.currentRange.pageIndex;
diff --git a/asterixdb/asterix-doc/pom.xml b/asterixdb/asterix-doc/pom.xml
index d0825d4..66a2856 100644
--- a/asterixdb/asterix-doc/pom.xml
+++ b/asterixdb/asterix-doc/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<url>http://asterixdb.apache.org/</url>
diff --git a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
index b237f30..af67e33 100644
--- a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
+++ b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
@@ -59,7 +59,7 @@
StreamGenerator::= FromClause LetClause? WhereClause? (GroupByClause LetClause? HavingClause?)?
-SelectClause ::= "SELECT" ("DISTINCT" | "ALL")? ( "VALUE" Expr | Projection ("," Projection)*)
+SelectClause ::= "SELECT" ("DISTINCT" | "ALL")? ( "VALUE" Expr | Projection ("," Projection)* ( "EXCLUDE" Identifier (("," | ".") Identifier)* )? )
Projection ::= (Expr ("AS"? Identifier)?) | (VariableRef "." "*") | "*"
@@ -162,7 +162,7 @@
CreateType ::= "CREATE" "TYPE" QualifiedName ("IF" "NOT" "EXISTS")? "AS" ObjectTypeDef
-ObjectTypeDef ::= ("CLOSED" | "OPEN")? "{" ObjectField ("," ObjectField)* "}"
+ObjectTypeDef ::= ("CLOSED" | "OPEN")? "{" ( ObjectField ("," ObjectField)* )? "}"
ObjectField ::= Identifier ":" Identifier "?"?
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
index 04a65d1..b8dc3bf 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
@@ -248,7 +248,7 @@
##### Example
-(Q3.5) Returns all of the different cities in the `customers` dataset.
+(Q3.5a) Returns all of the different cities in the `customers` dataset.
FROM customers AS c
SELECT DISTINCT c.address.city;
@@ -270,6 +270,32 @@
}
]
+### <a id="Select_exclude">SELECT EXCLUDE</a>
+The `EXCLUDE` keyword is used to remove one or more fields that would otherwise be returned from the `SELECT` clause.
+Conceptually, the scope of the `EXCLUDE` clause is the output of the `SELECT` clause itself.
+In a Stream Generator with both `DISTINCT` and `EXCLUDE` clauses, the `DISTINCT` clause is applied after the `EXCLUDE` clause.
+
+##### Example
+
+(Q3.5b) For the customer with `custid = C13`, return their information _excluding_ the `zipcode` field inside the `address` object and the top-level `name` field.
+
+ FROM customers AS c
+ WHERE c.custid = "C13"
+ SELECT c.* EXCLUDE address.zipcode, name;
+
+Result:
+
+ [
+ {
+ "custid": "C13",
+ "address": {
+ "street": "201 Main St.",
+ "city": "St. Louis, MO"
+ },
+ "rating": 750
+ }
+ ]
+
### <a id="Unnamed_projections">Unnamed Projections</a>
Similar to standard SQL, the query language supports unnamed projections (a.k.a, unnamed `SELECT` clause items), for which names are generated rather than user-provided.
diff --git a/asterixdb/asterix-docker/pom.xml b/asterixdb/asterix-docker/pom.xml
deleted file mode 100644
index 6c54337..0000000
--- a/asterixdb/asterix-docker/pom.xml
+++ /dev/null
@@ -1,68 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>apache-asterixdb</artifactId>
- <groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
- </parent>
- <artifactId>asterix-docker</artifactId>
-
- <properties>
- <root.dir>${basedir}/..</root.dir>
- </properties>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- <comments>A business-friendly OSS license</comments>
- </license>
- </licenses>
-
- <profiles>
- <profile>
- <id>docker</id>
- <build>
- <plugins>
- <plugin>
- <groupId>com.spotify</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <version>0.2.11</version>
- <configuration>
- <imageName>asterixdb/demo</imageName>
- <dockerDirectory>docker</dockerDirectory>
- <resources>
- <resource>
- <targetPath>/</targetPath>
- <directory>../asterix-server/target/</directory>
- <include>asterix-server-${project.version}-binary-assembly.zip</include>
- </resource>
- </resources>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
-</project>
diff --git a/asterixdb/asterix-examples/pom.xml b/asterixdb/asterix-examples/pom.xml
index 7812be0..0b47674 100644
--- a/asterixdb/asterix-examples/pom.xml
+++ b/asterixdb/asterix-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-examples</artifactId>
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index dec2ad8..14a3487 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
@@ -517,6 +517,10 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ </dependency>
<!-- apply patch for HADOOP-17225 to workaround CVE-2019-10172 -->
<dependency>
<groupId>org.codehaus.jackson</groupId>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalLangIPCProto.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalLangIPCProto.java
new file mode 100644
index 0000000..35e5961
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalLangIPCProto.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.ipc.MessageType;
+import org.apache.asterix.external.library.msgpack.MsgPackPointableVisitor;
+import org.apache.asterix.om.pointables.AFlatValuePointable;
+import org.apache.asterix.om.pointables.AListVisitablePointable;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeTagUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+public interface IExternalLangIPCProto {
+ static void visitValueRef(IAType type, DataOutput out, IValueReference valueReference,
+ PointableAllocator pointableAllocator, MsgPackPointableVisitor pointableVisitor, boolean visitNull)
+ throws IOException {
+ IVisitablePointable pointable;
+ switch (type.getTypeTag()) {
+ case OBJECT:
+ pointable = pointableAllocator.allocateRecordValue(type);
+ pointable.set(valueReference);
+ pointableVisitor.visit((ARecordVisitablePointable) pointable, pointableVisitor.getTypeInfo(type, out));
+ break;
+ case ARRAY:
+ case MULTISET:
+ pointable = pointableAllocator.allocateListValue(type);
+ pointable.set(valueReference);
+ pointableVisitor.visit((AListVisitablePointable) pointable, pointableVisitor.getTypeInfo(type, out));
+ break;
+ case ANY:
+ ATypeTag rtTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER
+ .deserialize(valueReference.getByteArray()[valueReference.getStartOffset()]);
+ IAType rtType = TypeTagUtil.getBuiltinTypeByTag(rtTypeTag);
+ visitValueRef(rtType, out, valueReference, pointableAllocator, pointableVisitor, visitNull);
+ break;
+ case MISSING:
+ case NULL:
+ if (!visitNull) {
+ return;
+ }
+ default:
+ pointable = pointableAllocator.allocateFieldValue(type);
+ pointable.set(valueReference);
+ pointableVisitor.visit((AFlatValuePointable) pointable, pointableVisitor.getTypeInfo(type, out));
+ break;
+ }
+ }
+
+ void start();
+
+ void helo() throws IOException, AsterixException;
+
+ long init(String module, String clazz, String fn) throws IOException, AsterixException;
+
+ ByteBuffer call(long functionId, IAType[] argTypes, IValueReference[] argValues, boolean nullCall)
+ throws IOException, AsterixException;
+
+ ByteBuffer callMulti(long key, ArrayBackedValueStorage args, int numTuples) throws IOException, AsterixException;
+
+ //For future use with interpreter reuse between jobs.
+ void quit() throws HyracksDataException;
+
+ void receiveMsg() throws IOException, AsterixException;
+
+ void sendHeader(long key, int msgLen) throws IOException;
+
+ void sendMsg(ArrayBackedValueStorage content) throws IOException;
+
+ void sendMsg() throws IOException;
+
+ MessageType getResponseType();
+
+ long getRouteId();
+
+ DataOutputStream getSockOut();
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILibraryEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILibraryEvaluator.java
new file mode 100644
index 0000000..8c6538b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILibraryEvaluator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.resources.IDeallocatable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+public interface ILibraryEvaluator extends IDeallocatable {
+
+ void start() throws IOException, AsterixException;
+
+ long initialize(IExternalFunctionInfo finfo) throws IOException, AsterixException;
+
+ ByteBuffer call(long id, IAType[] argTypes, IValueReference[] valueReferences, boolean nullCall) throws IOException;
+
+ ByteBuffer callMulti(long id, ArrayBackedValueStorage arguments, int numTuples) throws IOException;
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index f14af53..bbcf9cd 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.external.input.record.reader.aws;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
import java.io.IOException;
@@ -32,7 +31,7 @@
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.aws.s3.S3Utils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.CleanupUtils;
@@ -109,7 +108,7 @@
}
private boolean shouldRetry(String errorCode, int currentRetry) {
- return currentRetry < MAX_RETRIES && AwsS3.isRetryableError(errorCode);
+ return currentRetry < MAX_RETRIES && S3Utils.isRetryableError(errorCode);
}
@Override
@@ -134,7 +133,7 @@
private S3Client buildAwsS3Client(Map<String, String> configuration) throws HyracksDataException {
try {
- return ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
+ return S3Utils.buildAwsS3Client(configuration);
} catch (CompilationException ex) {
throw HyracksDataException.create(ex);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 89ea39e..a241354 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -26,6 +26,7 @@
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.aws.s3.S3Utils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -53,8 +54,7 @@
IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
//Get a list of S3 objects
- List<S3Object> filesOnly =
- ExternalDataUtils.AwsS3.listS3Objects(configuration, includeExcludeMatcher, warningCollector);
+ List<S3Object> filesOnly = S3Utils.listS3Objects(configuration, includeExcludeMatcher, warningCollector);
// Distribute work load amongst the partitions
distributeWorkLoad(filesOnly, getPartitionsCount());
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
index 4e3d1ec..92b7a95 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
@@ -32,6 +32,8 @@
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory.IncludeExcludeMatcher;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.aws.s3.S3Constants;
+import org.apache.asterix.external.util.aws.s3.S3Utils;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
@@ -61,7 +63,7 @@
try {
JobConf conf = createHdfsConf(serviceCtx, configuration);
int numberOfPartitions = getPartitionConstraint().getLocations().length;
- ExternalDataUtils.AwsS3.configureAwsS3HdfsJobConf(conf, configuration, numberOfPartitions);
+ S3Utils.configureAwsS3HdfsJobConf(conf, configuration, numberOfPartitions);
configureHdfsConf(conf, configuration);
} catch (SdkException | SdkBaseException ex) {
throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
@@ -107,8 +109,7 @@
throws CompilationException {
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
- List<S3Object> filesOnly =
- ExternalDataUtils.AwsS3.listS3Objects(configuration, includeExcludeMatcher, warningCollector);
+ List<S3Object> filesOnly = S3Utils.listS3Objects(configuration, includeExcludeMatcher, warningCollector);
StringBuilder builder = new StringBuilder();
if (!filesOnly.isEmpty()) {
@@ -123,7 +124,7 @@
}
private static void appendFileURI(StringBuilder builder, String container, S3Object file) {
- builder.append(ExternalDataConstants.AwsS3.HADOOP_S3_PROTOCOL);
+ builder.append(S3Constants.HADOOP_S3_PROTOCOL);
builder.append("://");
builder.append(container);
builder.append('/');
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStream.java
index cdb3834..bbfece2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStream.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.input.record.reader.azure.blob;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.buildAzureBlobClient;
import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
import java.io.IOException;
@@ -31,7 +32,6 @@
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.LogRedactionUtil;
@@ -86,7 +86,7 @@
private BlobServiceClient buildAzureClient(IApplicationContext appCtx, Map<String, String> configuration)
throws HyracksDataException {
try {
- return ExternalDataUtils.Azure.buildAzureBlobClient(appCtx, configuration);
+ return buildAzureBlobClient(appCtx, configuration);
} catch (CompilationException ex) {
throw HyracksDataException.create(ex);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java
index 064b319..55c0521 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.external.input.record.reader.azure.blob;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.buildAzureBlobClient;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.listBlobItems;
+
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -57,9 +60,9 @@
// Ensure the validity of include/exclude
ExternalDataUtils.validateIncludeExclude(configuration);
IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
- BlobServiceClient blobServiceClient = ExternalDataUtils.Azure.buildAzureBlobClient(appCtx, configuration);
- List<BlobItem> filesOnly = ExternalDataUtils.Azure.listBlobItems(blobServiceClient, configuration,
- includeExcludeMatcher, warningCollector);
+ BlobServiceClient blobServiceClient = buildAzureBlobClient(appCtx, configuration);
+ List<BlobItem> filesOnly =
+ listBlobItems(blobServiceClient, configuration, includeExcludeMatcher, warningCollector);
// Distribute work load amongst the partitions
distributeWorkLoad(filesOnly, getPartitionsCount());
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
index e34d188..7a95222 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.input.record.reader.azure.datalake;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.buildAzureDatalakeClient;
import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
import java.io.IOException;
@@ -31,7 +32,6 @@
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.LogRedactionUtil;
@@ -86,7 +86,7 @@
private DataLakeServiceClient buildAzureClient(IApplicationContext appCtx, Map<String, String> configuration)
throws HyracksDataException {
try {
- return ExternalDataUtils.Azure.buildAzureDatalakeClient(appCtx, configuration);
+ return buildAzureDatalakeClient(appCtx, configuration);
} catch (CompilationException ex) {
throw HyracksDataException.create(ex);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStreamFactory.java
index e9f8d4c..929cb6e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStreamFactory.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.external.input.record.reader.azure.datalake;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.buildAzureDatalakeClient;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.listDatalakePathItems;
+
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -57,9 +60,9 @@
// Ensure the validity of include/exclude
ExternalDataUtils.validateIncludeExclude(configuration);
IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
- DataLakeServiceClient client = ExternalDataUtils.Azure.buildAzureDatalakeClient(appCtx, configuration);
- List<PathItem> filesOnly = ExternalDataUtils.Azure.listDatalakePathItems(client, configuration,
- includeExcludeMatcher, warningCollector);
+ DataLakeServiceClient client = buildAzureDatalakeClient(appCtx, configuration);
+ List<PathItem> filesOnly =
+ listDatalakePathItems(client, configuration, includeExcludeMatcher, warningCollector);
// Distribute work load amongst the partitions
distributeWorkLoad(filesOnly, getPartitionsCount());
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java
index c2251df..e08013c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java
@@ -18,6 +18,11 @@
*/
package org.apache.asterix.external.input.record.reader.azure.parquet;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.HADOOP_AZURE_BLOB_PROTOCOL;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.buildAzureBlobClient;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.configureAzureHdfsJobConf;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.listBlobItems;
+
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -47,7 +52,7 @@
public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
IWarningCollector warningCollector) throws AlgebricksException, HyracksDataException {
IApplicationContext appCtx = (IApplicationContext) serviceCtx.getApplicationContext();
- BlobServiceClient blobServiceClient = ExternalDataUtils.Azure.buildAzureBlobClient(appCtx, configuration);
+ BlobServiceClient blobServiceClient = buildAzureBlobClient(appCtx, configuration);
//Get endpoint
String endPoint = extractEndPoint(blobServiceClient.getAccountUrl());
//Get path
@@ -57,7 +62,7 @@
//Configure Hadoop Azure input splits
JobConf conf = createHdfsConf(serviceCtx, configuration);
- ExternalDataUtils.Azure.configureAzureHdfsJobConf(conf, configuration, endPoint);
+ configureAzureHdfsJobConf(conf, configuration, endPoint);
configureHdfsConf(conf, configuration);
}
@@ -94,8 +99,8 @@
private static String buildPathURIs(Map<String, String> configuration, IWarningCollector warningCollector,
BlobServiceClient blobServiceClient, String endPoint) throws CompilationException {
IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
- List<BlobItem> filesOnly = ExternalDataUtils.Azure.listBlobItems(blobServiceClient, configuration,
- includeExcludeMatcher, warningCollector);
+ List<BlobItem> filesOnly =
+ listBlobItems(blobServiceClient, configuration, includeExcludeMatcher, warningCollector);
StringBuilder builder = new StringBuilder();
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
@@ -118,7 +123,7 @@
}
private static void appendFileURI(StringBuilder builder, String container, String endPoint, BlobItem file) {
- builder.append(ExternalDataConstants.Azure.HADOOP_AZURE_BLOB_PROTOCOL);
+ builder.append(HADOOP_AZURE_BLOB_PROTOCOL);
builder.append("://");
builder.append(container);
builder.append('@');
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureDataLakeParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureDataLakeParquetReaderFactory.java
index db87868..c98fc8b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureDataLakeParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureDataLakeParquetReaderFactory.java
@@ -18,6 +18,11 @@
*/
package org.apache.asterix.external.input.record.reader.azure.parquet;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.HADOOP_AZURE_DATALAKE_PROTOCOL;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.buildAzureDatalakeClient;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.configureAzureHdfsJobConf;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.listDatalakePathItems;
+
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -47,8 +52,7 @@
public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
IWarningCollector warningCollector) throws AlgebricksException, HyracksDataException {
IApplicationContext appCtx = (IApplicationContext) serviceCtx.getApplicationContext();
- DataLakeServiceClient dataLakeServiceClient =
- ExternalDataUtils.Azure.buildAzureDatalakeClient(appCtx, configuration);
+ DataLakeServiceClient dataLakeServiceClient = buildAzureDatalakeClient(appCtx, configuration);
//Get endpoint
String endPoint = extractEndPoint(dataLakeServiceClient.getAccountUrl());
@@ -61,7 +65,7 @@
//Configure Hadoop Azure input splits
JobConf conf = createHdfsConf(serviceCtx, configuration);
- ExternalDataUtils.Azure.configureAzureHdfsJobConf(conf, configuration, endPoint);
+ configureAzureHdfsJobConf(conf, configuration, endPoint);
configureHdfsConf(conf, configuration);
}
@@ -98,8 +102,8 @@
private static String buildPathURIs(Map<String, String> configuration, IWarningCollector warningCollector,
DataLakeServiceClient dataLakeServiceClient, String endPoint) throws CompilationException {
IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
- List<PathItem> filesOnly = ExternalDataUtils.Azure.listDatalakePathItems(dataLakeServiceClient, configuration,
- includeExcludeMatcher, warningCollector);
+ List<PathItem> filesOnly =
+ listDatalakePathItems(dataLakeServiceClient, configuration, includeExcludeMatcher, warningCollector);
StringBuilder builder = new StringBuilder();
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
@@ -122,7 +126,7 @@
}
private static void appendFileURI(StringBuilder builder, String container, String endPoint, PathItem file) {
- builder.append(ExternalDataConstants.Azure.HADOOP_AZURE_DATALAKE_PROTOCOL);
+ builder.append(HADOOP_AZURE_DATALAKE_PROTOCOL);
builder.append("://");
builder.append(container);
builder.append('@');
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
index 5da4583..4657bd0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
@@ -32,7 +32,7 @@
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.google.gcs.GCSUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.CleanupUtils;
@@ -134,7 +134,7 @@
private Storage buildClient(Map<String, String> configuration) throws HyracksDataException {
try {
- return ExternalDataUtils.GCS.buildClient(configuration);
+ return GCSUtils.buildClient(configuration);
} catch (CompilationException ex) {
throw HyracksDataException.create(ex);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
index 0e7ea90..278c1ad 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
@@ -19,33 +19,23 @@
package org.apache.asterix.external.input.record.reader.gcs;
import static org.apache.asterix.external.util.ExternalDataUtils.getIncludeExcludeMatchers;
-import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
-import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
-import java.util.function.BiPredicate;
-import java.util.regex.Matcher;
-import org.apache.asterix.common.exceptions.CompilationException;
-import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.google.gcs.GCSUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.Warning;
-import com.google.api.gax.paging.Page;
-import com.google.cloud.BaseServiceException;
import com.google.cloud.storage.Blob;
-import com.google.cloud.storage.Storage;
public class GCSInputStreamFactory extends AbstractExternalInputStreamFactory {
@@ -63,57 +53,16 @@
// Ensure the validity of include/exclude
ExternalDataUtils.validateIncludeExclude(configuration);
-
- // Prepare to retrieve the objects
- List<Blob> filesOnly = new ArrayList<>();
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- Storage gcs = ExternalDataUtils.GCS.buildClient(configuration);
- Storage.BlobListOption options = Storage.BlobListOption.prefix(ExternalDataUtils.getPrefix(configuration));
- Page<Blob> items;
-
- try {
- items = gcs.list(container, options);
- } catch (BaseServiceException ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
-
- // Collect the paths to files only
IncludeExcludeMatcher includeExcludeMatcher = getIncludeExcludeMatchers(configuration);
- collectAndFilterFiles(items, includeExcludeMatcher.getPredicate(), includeExcludeMatcher.getMatchersList(),
- filesOnly);
- // Warn if no files are returned
- if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
- Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- warningCollector.warn(warning);
- }
+ // get the items
+ List<Blob> filesOnly = GCSUtils.listItems(configuration, includeExcludeMatcher, warningCollector);
// Distribute work load amongst the partitions
distributeWorkLoad(filesOnly, getPartitionsCount());
}
/**
- * AWS S3 returns all the objects as paths, not differentiating between folder and files. The path is considered
- * a file if it does not end up with a "/" which is the separator in a folder structure.
- *
- * @param items List of returned objects
- */
- private void collectAndFilterFiles(Page<Blob> items, BiPredicate<List<Matcher>, String> predicate,
- List<Matcher> matchers, List<Blob> filesOnly) {
- for (Blob item : items.iterateAll()) {
- // skip folders
- if (item.getName().endsWith("/")) {
- continue;
- }
-
- // No filter, add file
- if (predicate.test(matchers, item.getName())) {
- filesOnly.add(item);
- }
- }
- }
-
- /**
* To efficiently utilize the parallelism, work load will be distributed amongst the partitions based on the file
* size.
*
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
new file mode 100644
index 0000000..2887415
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.gcs.parquet;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.external.input.HDFSDataSourceFactory;
+import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory.IncludeExcludeMatcher;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.google.gcs.GCSConstants;
+import org.apache.asterix.external.util.google.gcs.GCSUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.application.IServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+
+import com.google.cloud.storage.Blob;
+
+public class GCSParquetReaderFactory extends HDFSDataSourceFactory {
+ private static final long serialVersionUID = -6140824803254158253L;
+ private static final List<String> recordReaderNames =
+ Collections.singletonList(ExternalDataConstants.KEY_ADAPTER_NAME_GCS);
+
+ @Override
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws AlgebricksException, HyracksDataException {
+ // get path
+ String path = buildPathURIs(configuration, warningCollector);
+
+ // put GCS configurations to AsterixDB's Hadoop configuration
+ putGCSConfToHadoopConf(configuration, path);
+
+ // configure hadoop input splits
+ JobConf conf = createHdfsConf(serviceCtx, configuration);
+ int numberOfPartitions = getPartitionConstraint().getLocations().length;
+ GCSUtils.configureHdfsJobConf(conf, configuration, numberOfPartitions);
+ configureHdfsConf(conf, configuration);
+ }
+
+ @Override
+ public List<String> getRecordReaderNames() {
+ return recordReaderNames;
+ }
+
+ @Override
+ public Set<String> getReaderSupportedFormats() {
+ return Collections.singleton(ExternalDataConstants.FORMAT_PARQUET);
+ }
+
+ /**
+ * Prepare Hadoop configurations to read parquet files
+ *
+ * @param path Comma-delimited paths
+ */
+ private static void putGCSConfToHadoopConf(Map<String, String> configuration, String path) {
+ configuration.put(ExternalDataConstants.KEY_PATH, path);
+ configuration.put(ExternalDataConstants.KEY_INPUT_FORMAT, ExternalDataConstants.INPUT_FORMAT_PARQUET);
+ configuration.put(ExternalDataConstants.KEY_PARSER, ExternalDataConstants.FORMAT_NOOP);
+ }
+
+ /**
+ * Build Google Cloud Storage path-style for the requested files
+ *
+ * @param configuration properties
+ * @param warningCollector warning collector
+ * @return Comma-delimited paths (e.g., "gs://bucket/file1.parquet,gs://bucket/file2.parquet")
+ * @throws CompilationException Compilation exception
+ */
+ private static String buildPathURIs(Map<String, String> configuration, IWarningCollector warningCollector)
+ throws CompilationException {
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+
+ // Ensure the validity of include/exclude
+ ExternalDataUtils.validateIncludeExclude(configuration);
+ IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
+
+ // get the items
+ List<Blob> filesOnly = GCSUtils.listItems(configuration, includeExcludeMatcher, warningCollector);
+ StringBuilder builder = new StringBuilder();
+
+ if (!filesOnly.isEmpty()) {
+ appendFileURI(builder, container, filesOnly.get(0));
+ for (int i = 1; i < filesOnly.size(); i++) {
+ builder.append(',');
+ appendFileURI(builder, container, filesOnly.get(i));
+ }
+ }
+
+ return builder.toString();
+ }
+
+ private static void appendFileURI(StringBuilder builder, String container, Blob file) {
+ builder.append(GCSConstants.HADOOP_GCS_PROTOCOL);
+ builder.append("://");
+ builder.append(container);
+ builder.append('/');
+ builder.append(file.getName());
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
index 7258359..af8733f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
@@ -34,7 +34,7 @@
import org.apache.asterix.om.types.AbstractCollectionType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.IATypeVisitor;
-import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
import org.apache.asterix.runtime.projection.FunctionCallInformation;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
@@ -67,9 +67,9 @@
public MessageType clipType(ARecordType rootType, MessageType fileSchema,
Map<String, FunctionCallInformation> funcInfo) {
- if (rootType == DataProjectionInfo.EMPTY_TYPE) {
+ if (rootType == DataProjectionFiltrationInfo.EMPTY_TYPE) {
return EMPTY_PARQUET_MESSAGE;
- } else if (rootType == DataProjectionInfo.ALL_FIELDS_TYPE) {
+ } else if (rootType == DataProjectionFiltrationInfo.ALL_FIELDS_TYPE) {
return fileSchema;
}
Types.MessageTypeBuilder builder = Types.buildMessage();
@@ -174,7 +174,7 @@
//If no warning is created, then it means it has been reported
Warning warning = null;
if (actualType != ATypeTag.SYSTEM_NULL) {
- warning = info.createTypeMismatchWarning(expectedType, actualType);
+ warning = info.createWarning(expectedType, actualType);
}
if (warning != null) {
//New warning that we saw for the first time. We should report it.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/AbstractPythonIPCProto.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/AbstractPythonIPCProto.java
new file mode 100644
index 0000000..00d1dcc
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/AbstractPythonIPCProto.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.ipc;
+
+import static org.apache.hyracks.ipc.impl.Message.HEADER_SIZE;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.api.IExternalLangIPCProto;
+import org.apache.asterix.external.library.msgpack.MsgPackPointableVisitor;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.ipc.impl.Message;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessageUnpacker;
+import org.msgpack.core.buffer.ArrayBufferInput;
+
+public abstract class AbstractPythonIPCProto {
+ public static final int HEADER_SIZE_LEN_INCLUSIVE = 21;
+ protected final PythonMessageBuilder messageBuilder;
+ protected final DataOutputStream sockOut;
+ protected final ArrayBufferInput unpackerInput;
+ protected final MessageUnpacker unpacker;
+ protected final ArrayBackedValueStorage argsStorage;
+ protected final PointableAllocator pointableAllocator;
+ protected final MsgPackPointableVisitor pointableVisitor;
+ private final ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_SIZE_LEN_INCLUSIVE);
+ protected ByteBuffer recvBuffer = ByteBuffer.allocate(32768);
+ protected long routeId;
+ protected Pair<ByteBuffer, Exception> bufferBox;
+ protected long maxFunctionId;
+
+ public AbstractPythonIPCProto(OutputStream sockOut) {
+ messageBuilder = new PythonMessageBuilder();
+ this.sockOut = new DataOutputStream(sockOut);
+ this.maxFunctionId = 0L;
+ unpackerInput = new ArrayBufferInput(new byte[0]);
+ unpacker = MessagePack.newDefaultUnpacker(unpackerInput);
+ this.argsStorage = new ArrayBackedValueStorage();
+ this.pointableAllocator = new PointableAllocator();
+ this.pointableVisitor = new MsgPackPointableVisitor();
+ }
+
+ public void helo() throws IOException, AsterixException {
+ recvBuffer.clear();
+ recvBuffer.position(0);
+ recvBuffer.limit(0);
+ messageBuilder.reset();
+ messageBuilder.hello();
+ sendHeader(routeId, messageBuilder.getLength());
+ sendMsg();
+ receiveMsg();
+ if (getResponseType() != MessageType.HELO) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
+ "Expected HELO, recieved " + getResponseType().name());
+ }
+ }
+
+ public long init(String module, String clazz, String fn) throws IOException, AsterixException {
+ long functionId = maxFunctionId++;
+ recvBuffer.clear();
+ recvBuffer.position(0);
+ recvBuffer.limit(0);
+ messageBuilder.reset();
+ messageBuilder.init(module, clazz, fn);
+ sendHeader(functionId, messageBuilder.getLength());
+ sendMsg();
+ receiveMsg();
+ if (getResponseType() != MessageType.INIT_RSP) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
+ "Expected INIT_RSP, recieved " + getResponseType().name());
+ }
+ return functionId;
+ }
+
+ public ByteBuffer call(long functionId, IAType[] argTypes, IValueReference[] argValues, boolean nullCall)
+ throws IOException, AsterixException {
+ recvBuffer.clear();
+ recvBuffer.position(0);
+ recvBuffer.limit(0);
+ messageBuilder.reset();
+ argsStorage.reset();
+ for (int i = 0; i < argTypes.length; i++) {
+ IExternalLangIPCProto.visitValueRef(argTypes[i], argsStorage.getDataOutput(), argValues[i],
+ pointableAllocator, pointableVisitor, nullCall);
+ }
+ int len = argsStorage.getLength() + 5;
+ sendHeader(functionId, len);
+ messageBuilder.call(argValues.length, len);
+ sendMsg(argsStorage);
+ receiveMsg();
+ if (getResponseType() != MessageType.CALL_RSP) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
+ "Expected CALL_RSP, recieved " + getResponseType().name());
+ }
+ return recvBuffer;
+ }
+
+ public ByteBuffer callMulti(long key, ArrayBackedValueStorage args, int numTuples)
+ throws IOException, AsterixException {
+ recvBuffer.clear();
+ recvBuffer.position(0);
+ recvBuffer.limit(0);
+ messageBuilder.reset();
+ int len = args.getLength() + 4;
+ sendHeader(key, len);
+ messageBuilder.callMulti(0, numTuples);
+ sendMsg(args);
+ receiveMsg();
+ if (getResponseType() != MessageType.CALL_RSP) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
+ "Expected CALL_RSP, recieved " + getResponseType().name());
+ }
+ return recvBuffer;
+ }
+
+ public void quit() throws HyracksDataException {
+ messageBuilder.quit();
+ }
+
+ public abstract void receiveMsg() throws IOException, AsterixException;
+
+ public void sendHeader(long key, int msgLen) throws IOException {
+ headerBuffer.clear();
+ headerBuffer.position(0);
+ headerBuffer.putInt(HEADER_SIZE + Integer.BYTES + msgLen);
+ headerBuffer.putLong(key);
+ headerBuffer.putLong(routeId);
+ headerBuffer.put(Message.NORMAL);
+ sockOut.write(headerBuffer.array(), 0, HEADER_SIZE + Integer.BYTES);
+ sockOut.flush();
+ }
+
+ public void sendMsg(ArrayBackedValueStorage content) throws IOException {
+ sockOut.write(messageBuilder.getBuf().array(), messageBuilder.getBuf().arrayOffset(),
+ messageBuilder.getBuf().position());
+ sockOut.write(content.getByteArray(), content.getStartOffset(), content.getLength());
+ sockOut.flush();
+ }
+
+ public void sendMsg() throws IOException {
+ sockOut.write(messageBuilder.getBuf().array(), messageBuilder.getBuf().arrayOffset(),
+ messageBuilder.getBuf().position());
+ sockOut.flush();
+ }
+
+ public MessageType getResponseType() {
+ return messageBuilder.type;
+ }
+
+ public long getRouteId() {
+ return routeId;
+ }
+
+ public DataOutputStream getSockOut() {
+ return sockOut;
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonDomainSocketProto.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonDomainSocketProto.java
new file mode 100644
index 0000000..89f240a
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonDomainSocketProto.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.ipc;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.api.IExternalLangIPCProto;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.msgpack.core.MessagePack;
+
+public class PythonDomainSocketProto extends AbstractPythonIPCProto implements IExternalLangIPCProto {
+ private final String wd;
+ SocketChannel chan;
+ private ByteBuffer headerBuffer;
+ private ProcessHandle pid;
+ public static final int HYR_HEADER_SIZE = 21; // 4 (sz) + 8 (mid) + 8 (rmid) + 1 (flags)
+ public static final int HYR_HEADER_SIZE_NOSZ = 17; // 8 + 8 + 1
+
+ public PythonDomainSocketProto(OutputStream sockOut, SocketChannel chan, String wd) {
+ super(sockOut);
+ this.chan = chan;
+ this.wd = wd;
+ headerBuffer = ByteBuffer.allocate(HYR_HEADER_SIZE);
+ }
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void helo() throws IOException, AsterixException {
+ recvBuffer.clear();
+ recvBuffer.position(0);
+ recvBuffer.limit(0);
+ messageBuilder.reset();
+ messageBuilder.helloDS(wd);
+ sendHeader(routeId, messageBuilder.getLength());
+ sendMsg(true);
+ receiveMsg(true);
+ byte pidType = recvBuffer.get();
+ if (pidType != MessagePack.Code.UINT32 && pidType != MessagePack.Code.UINT16) {
+ throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ "Returned pid type is incorrect: " + pidType);
+ }
+ switch (pidType) {
+ case MessagePack.Code.UINT32:
+ pid = ProcessHandle.of(recvBuffer.getInt()).get();
+ break;
+ case MessagePack.Code.UINT16:
+ pid = ProcessHandle.of(recvBuffer.getShort()).get();
+ break;
+ case MessagePack.Code.UINT8:
+ pid = ProcessHandle.of(recvBuffer.get()).get();
+ break;
+ default:
+ throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ "Returned pid type is incorrect: " + pidType);
+ }
+ if (getResponseType() != MessageType.HELO) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
+ "Expected HELO, recieved " + getResponseType().name());
+ }
+ }
+
+ @Override
+ public void sendMsg() throws IOException {
+ sendMsg(false);
+ }
+
+ @Override
+ public void sendMsg(ArrayBackedValueStorage args) throws IOException {
+ sendMsg(false, args);
+ }
+
+ public void sendMsg(boolean sendIfDead) throws IOException {
+ if (!sendIfDead && (pid == null || !pid.isAlive())) {
+ return;
+ }
+ super.sendMsg();
+ }
+
+ public void sendMsg(boolean sendIfDead, ArrayBackedValueStorage args) throws IOException {
+ if (!sendIfDead && (pid == null || !pid.isAlive())) {
+ return;
+ }
+ super.sendMsg(args);
+ }
+
+ @Override
+ public void receiveMsg() throws IOException, AsterixException {
+ receiveMsg(false);
+ }
+
+ public void receiveMsg(boolean sendIfDead) throws IOException, AsterixException {
+ if (!sendIfDead && (pid == null || !pid.isAlive())) {
+ throw new AsterixException("Python process exited unexpectedly");
+ }
+ readFully(headerBuffer.capacity(), headerBuffer);
+ if (headerBuffer.remaining() < Integer.BYTES) {
+ recvBuffer.limit(0);
+ throw new AsterixException("Python process exited unexpectedly");
+ }
+ int msgSz = headerBuffer.getInt() - HYR_HEADER_SIZE_NOSZ;
+ if (recvBuffer.capacity() < msgSz) {
+ recvBuffer = ByteBuffer.allocate(((msgSz / 32768) + 1) * 32768);
+ }
+ readFully(msgSz, recvBuffer);
+ messageBuilder.readHead(recvBuffer);
+ if (messageBuilder.type == MessageType.ERROR) {
+ unpackerInput.reset(recvBuffer.array(), recvBuffer.position() + recvBuffer.arrayOffset(),
+ recvBuffer.remaining());
+ unpacker.reset(unpackerInput);
+ throw new AsterixException(unpacker.unpackString().replace('\0', ' '));
+ }
+ }
+
+ private void readFully(int msgSz, ByteBuffer buf) throws IOException, AsterixException {
+ buf.limit(msgSz);
+ buf.clear();
+ int read;
+ int size = msgSz;
+ while (size > 0) {
+ read = chan.read(buf);
+ if (read < 0) {
+ throw new AsterixException("Socket closed");
+ }
+ size -= read;
+ }
+ buf.flip();
+ }
+
+ @Override
+ public void quit() throws HyracksDataException {
+ messageBuilder.quit();
+ }
+
+ public ProcessHandle getPid() {
+ return pid;
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonIPCProto.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonIPCProto.java
deleted file mode 100644
index c803517..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonIPCProto.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.ipc;
-
-import static org.apache.hyracks.ipc.impl.Message.HEADER_SIZE;
-
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.exceptions.ErrorCode;
-import org.apache.asterix.external.library.msgpack.MsgPackPointableVisitor;
-import org.apache.asterix.om.pointables.AFlatValuePointable;
-import org.apache.asterix.om.pointables.AListVisitablePointable;
-import org.apache.asterix.om.pointables.ARecordVisitablePointable;
-import org.apache.asterix.om.pointables.PointableAllocator;
-import org.apache.asterix.om.pointables.base.IVisitablePointable;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.TypeTagUtil;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IValueReference;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.ipc.impl.Message;
-import org.msgpack.core.MessagePack;
-import org.msgpack.core.MessageUnpacker;
-import org.msgpack.core.buffer.ArrayBufferInput;
-
-public class PythonIPCProto {
-
- private final PythonMessageBuilder messageBuilder;
- private final DataOutputStream sockOut;
- private final ByteBuffer headerBuffer = ByteBuffer.allocate(21);
- private ByteBuffer recvBuffer = ByteBuffer.allocate(32768);
- private final ExternalFunctionResultRouter router;
- private long routeId;
- private Pair<ByteBuffer, Exception> bufferBox;
- private final Process pythonProc;
- private long maxFunctionId;
- private final ArrayBufferInput unpackerInput;
- private final MessageUnpacker unpacker;
- private final ArrayBackedValueStorage argsStorage;
- private final PointableAllocator pointableAllocator;
- private final MsgPackPointableVisitor pointableVisitor;
-
- public PythonIPCProto(OutputStream sockOut, ExternalFunctionResultRouter router, Process pythonProc) {
- this.sockOut = new DataOutputStream(sockOut);
- messageBuilder = new PythonMessageBuilder();
- this.router = router;
- this.pythonProc = pythonProc;
- this.maxFunctionId = 0L;
- unpackerInput = new ArrayBufferInput(new byte[0]);
- unpacker = MessagePack.newDefaultUnpacker(unpackerInput);
- this.argsStorage = new ArrayBackedValueStorage();
- this.pointableAllocator = new PointableAllocator();
- this.pointableVisitor = new MsgPackPointableVisitor();
- }
-
- public void start() {
- Pair<Long, Pair<ByteBuffer, Exception>> keyAndBufferBox = router.insertRoute(recvBuffer);
- this.routeId = keyAndBufferBox.getFirst();
- this.bufferBox = keyAndBufferBox.getSecond();
- }
-
- public void helo() throws IOException, AsterixException {
- recvBuffer.clear();
- recvBuffer.position(0);
- recvBuffer.limit(0);
- messageBuilder.reset();
- messageBuilder.hello();
- sendHeader(routeId, messageBuilder.getLength());
- sendMsg();
- receiveMsg();
- if (getResponseType() != MessageType.HELO) {
- throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
- "Expected HELO, recieved " + getResponseType().name());
- }
- }
-
- public long init(String module, String clazz, String fn) throws IOException, AsterixException {
- long functionId = maxFunctionId++;
- recvBuffer.clear();
- recvBuffer.position(0);
- recvBuffer.limit(0);
- messageBuilder.reset();
- messageBuilder.init(module, clazz, fn);
- sendHeader(functionId, messageBuilder.getLength());
- sendMsg();
- receiveMsg();
- if (getResponseType() != MessageType.INIT_RSP) {
- throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
- "Expected INIT_RSP, recieved " + getResponseType().name());
- }
- return functionId;
- }
-
- public ByteBuffer call(long functionId, IAType[] argTypes, IValueReference[] argValues, boolean nullCall)
- throws IOException, AsterixException {
- recvBuffer.clear();
- recvBuffer.position(0);
- recvBuffer.limit(0);
- messageBuilder.reset();
- argsStorage.reset();
- for (int i = 0; i < argTypes.length; i++) {
- visitValueRef(argTypes[i], argsStorage.getDataOutput(), argValues[i], pointableAllocator, pointableVisitor,
- nullCall);
- }
- int len = argsStorage.getLength() + 5;
- sendHeader(functionId, len);
- messageBuilder.call(argValues.length, len);
- sendMsg(argsStorage);
- receiveMsg();
- if (getResponseType() != MessageType.CALL_RSP) {
- throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
- "Expected CALL_RSP, recieved " + getResponseType().name());
- }
- return recvBuffer;
- }
-
- public ByteBuffer callMulti(long key, ArrayBackedValueStorage args, int numTuples)
- throws IOException, AsterixException {
- recvBuffer.clear();
- recvBuffer.position(0);
- recvBuffer.limit(0);
- messageBuilder.reset();
- int len = args.getLength() + 4;
- sendHeader(key, len);
- messageBuilder.callMulti(0, numTuples);
- sendMsg(args);
- receiveMsg();
- if (getResponseType() != MessageType.CALL_RSP) {
- throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
- "Expected CALL_RSP, recieved " + getResponseType().name());
- }
- return recvBuffer;
- }
-
- //For future use with interpreter reuse between jobs.
- public void quit() throws HyracksDataException {
- messageBuilder.quit();
- router.removeRoute(routeId);
- }
-
- public void receiveMsg() throws IOException, AsterixException {
- Exception except;
- try {
- synchronized (bufferBox) {
- while ((bufferBox.getFirst().limit() == 0 || bufferBox.getSecond() != null) && pythonProc.isAlive()) {
- bufferBox.wait(100);
- }
- }
- except = router.getAndRemoveException(routeId);
- if (!pythonProc.isAlive()) {
- except = new IOException("Python process exited with code: " + pythonProc.exitValue());
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new AsterixException(ErrorCode.EXTERNAL_UDF_EXCEPTION, e);
- }
- if (except != null) {
- throw new AsterixException(except);
- }
- if (bufferBox.getFirst() != recvBuffer) {
- recvBuffer = bufferBox.getFirst();
- }
- messageBuilder.readHead(recvBuffer);
- if (messageBuilder.type == MessageType.ERROR) {
- unpackerInput.reset(recvBuffer.array(), recvBuffer.position() + recvBuffer.arrayOffset(),
- recvBuffer.remaining());
- unpacker.reset(unpackerInput);
- throw new AsterixException(unpacker.unpackString());
- }
- }
-
- public void sendHeader(long key, int msgLen) throws IOException {
- headerBuffer.clear();
- headerBuffer.position(0);
- headerBuffer.putInt(HEADER_SIZE + Integer.BYTES + msgLen);
- headerBuffer.putLong(key);
- headerBuffer.putLong(routeId);
- headerBuffer.put(Message.NORMAL);
- sockOut.write(headerBuffer.array(), 0, HEADER_SIZE + Integer.BYTES);
- sockOut.flush();
- }
-
- public void sendMsg(ArrayBackedValueStorage content) throws IOException {
- sockOut.write(messageBuilder.getBuf().array(), messageBuilder.getBuf().arrayOffset(),
- messageBuilder.getBuf().position());
- sockOut.write(content.getByteArray(), content.getStartOffset(), content.getLength());
- sockOut.flush();
- }
-
- public void sendMsg() throws IOException {
- sockOut.write(messageBuilder.getBuf().array(), messageBuilder.getBuf().arrayOffset(),
- messageBuilder.getBuf().position());
- sockOut.flush();
- }
-
- public MessageType getResponseType() {
- return messageBuilder.type;
- }
-
- public long getRouteId() {
- return routeId;
- }
-
- public DataOutputStream getSockOut() {
- return sockOut;
- }
-
- public static void visitValueRef(IAType type, DataOutput out, IValueReference valueReference,
- PointableAllocator pointableAllocator, MsgPackPointableVisitor pointableVisitor, boolean visitNull)
- throws IOException {
- IVisitablePointable pointable;
- switch (type.getTypeTag()) {
- case OBJECT:
- pointable = pointableAllocator.allocateRecordValue(type);
- pointable.set(valueReference);
- pointableVisitor.visit((ARecordVisitablePointable) pointable, pointableVisitor.getTypeInfo(type, out));
- break;
- case ARRAY:
- case MULTISET:
- pointable = pointableAllocator.allocateListValue(type);
- pointable.set(valueReference);
- pointableVisitor.visit((AListVisitablePointable) pointable, pointableVisitor.getTypeInfo(type, out));
- break;
- case ANY:
- ATypeTag rtTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER
- .deserialize(valueReference.getByteArray()[valueReference.getStartOffset()]);
- IAType rtType = TypeTagUtil.getBuiltinTypeByTag(rtTypeTag);
- switch (rtTypeTag) {
- case OBJECT:
- pointable = pointableAllocator.allocateRecordValue(rtType);
- pointable.set(valueReference);
- pointableVisitor.visit((ARecordVisitablePointable) pointable,
- pointableVisitor.getTypeInfo(rtType, out));
- break;
- case ARRAY:
- case MULTISET:
- pointable = pointableAllocator.allocateListValue(rtType);
- pointable.set(valueReference);
- pointableVisitor.visit((AListVisitablePointable) pointable,
- pointableVisitor.getTypeInfo(rtType, out));
- break;
- case MISSING:
- case NULL:
- if (!visitNull) {
- return;
- }
- default:
- pointable = pointableAllocator.allocateFieldValue(rtType);
- pointable.set(valueReference);
- pointableVisitor.visit((AFlatValuePointable) pointable,
- pointableVisitor.getTypeInfo(rtType, out));
- break;
- }
- break;
- case MISSING:
- case NULL:
- if (!visitNull) {
- return;
- }
- default:
- pointable = pointableAllocator.allocateFieldValue(type);
- pointable.set(valueReference);
- pointableVisitor.visit((AFlatValuePointable) pointable, pointableVisitor.getTypeInfo(type, out));
- break;
- }
- }
-
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonMessageBuilder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonMessageBuilder.java
index 5429657..20f8306 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonMessageBuilder.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonMessageBuilder.java
@@ -82,6 +82,16 @@
buf.put(serAddr);
}
+ public void helloDS(String modulePath) throws IOException {
+ this.type = MessageType.HELO;
+ // sum(string lengths) + 2 from fix array tag and message type
+ dataLength = PythonMessageBuilder.getStringLength(modulePath) + 2;
+ packHeader();
+ MessagePackUtils.packFixArrayHeader(buf, (byte) 2);
+ MessagePackUtils.packStr(buf, "HELLO");
+ MessagePackUtils.packStr(buf, modulePath);
+ }
+
public void quit() throws HyracksDataException {
this.type = MessageType.QUIT;
dataLength = getStringLength("QUIT");
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonTCPSocketProto.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonTCPSocketProto.java
new file mode 100644
index 0000000..7fd3de4
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/ipc/PythonTCPSocketProto.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.ipc;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.api.IExternalLangIPCProto;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class PythonTCPSocketProto extends AbstractPythonIPCProto
+ implements org.apache.asterix.external.api.IExternalLangIPCProto {
+
+ private final ExternalFunctionResultRouter router;
+ private final Process proc;
+
+ public PythonTCPSocketProto(OutputStream sockOut, ExternalFunctionResultRouter router, Process pythonProc) {
+ super(sockOut);
+ this.router = router;
+ this.proc = pythonProc;
+ }
+
+ @Override
+ public void start() {
+ Pair<Long, Pair<ByteBuffer, Exception>> keyAndBufferBox = router.insertRoute(recvBuffer);
+ this.routeId = keyAndBufferBox.getFirst();
+ this.bufferBox = keyAndBufferBox.getSecond();
+ }
+
+ @Override
+ public void quit() throws HyracksDataException {
+ messageBuilder.quit();
+ router.removeRoute(routeId);
+ }
+
+ @Override
+ public void receiveMsg() throws IOException, AsterixException {
+ Exception except;
+ try {
+ synchronized (bufferBox) {
+ while ((bufferBox.getFirst().limit() == 0 || bufferBox.getSecond() != null) && proc.isAlive()) {
+ bufferBox.wait(100);
+ }
+ }
+ except = router.getAndRemoveException(routeId);
+ if (!proc.isAlive()) {
+ except = new IOException("Python process exited with code: " + proc.exitValue());
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AsterixException(ErrorCode.EXTERNAL_UDF_EXCEPTION, e);
+ }
+ if (except != null) {
+ throw new AsterixException(except);
+ }
+ if (bufferBox.getFirst() != recvBuffer) {
+ recvBuffer = bufferBox.getFirst();
+ }
+ messageBuilder.readHead(recvBuffer);
+ if (messageBuilder.type == MessageType.ERROR) {
+ unpackerInput.reset(recvBuffer.array(), recvBuffer.position() + recvBuffer.arrayOffset(),
+ recvBuffer.remaining());
+ unpacker.reset(unpackerInput);
+ throw new AsterixException(unpacker.unpackString());
+ }
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/AbstractLibrarySocketEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/AbstractLibrarySocketEvaluator.java
new file mode 100644
index 0000000..6fcfdcf
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/AbstractLibrarySocketEvaluator.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.library;
+
+import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_UDF_EXCEPTION;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.IExternalLangIPCProto;
+import org.apache.asterix.external.api.ILibraryEvaluator;
+import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.dataflow.TaskAttemptId;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.std.base.AbstractStateObject;
+
+public abstract class AbstractLibrarySocketEvaluator extends AbstractStateObject implements ILibraryEvaluator {
+
+ protected IExternalLangIPCProto proto;
+ protected TaskAttemptId task;
+ protected IWarningCollector warningCollector;
+ protected SourceLocation sourceLoc;
+
+ public AbstractLibrarySocketEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId, TaskAttemptId task,
+ IWarningCollector warningCollector, SourceLocation sourceLoc) {
+ super(jobId, evaluatorId);
+ this.task = task;
+ this.warningCollector = warningCollector;
+ this.sourceLoc = sourceLoc;
+ }
+
+ @Override
+ public long initialize(IExternalFunctionInfo finfo) throws IOException, AsterixException {
+ List<String> externalIdents = finfo.getExternalIdentifier();
+ String packageModule = externalIdents.get(0);
+ String clazz;
+ String fn;
+ String externalIdent1 = externalIdents.get(1);
+ int idx = externalIdent1.lastIndexOf('.');
+ if (idx >= 0) {
+ clazz = externalIdent1.substring(0, idx);
+ fn = externalIdent1.substring(idx + 1);
+ } else {
+ clazz = null;
+ fn = externalIdent1;
+ }
+ return proto.init(packageModule, clazz, fn);
+ }
+
+ @Override
+ public ByteBuffer call(long id, IAType[] argTypes, IValueReference[] valueReferences, boolean nullCall)
+ throws IOException {
+ ByteBuffer ret = null;
+ try {
+ ret = proto.call(id, argTypes, valueReferences, nullCall);
+ } catch (AsterixException e) {
+ if (warningCollector.shouldWarn()) {
+ warningCollector.warn(Warning.of(sourceLoc, EXTERNAL_UDF_EXCEPTION, e.getMessage()));
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public ByteBuffer callMulti(long id, ArrayBackedValueStorage arguments, int numTuples) throws IOException {
+ ByteBuffer ret = null;
+ try {
+ ret = proto.callMulti(id, arguments, numTuples);
+ } catch (AsterixException e) {
+ if (warningCollector.shouldWarn()) {
+ warningCollector.warn(Warning.of(sourceLoc, EXTERNAL_UDF_EXCEPTION, e.getMessage()));
+ }
+ }
+ return ret;
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
index 94a4dd2..fb8d761 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -28,6 +28,7 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.api.ILibraryEvaluator;
import org.apache.asterix.external.library.msgpack.MessageUnpackerToADM;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
@@ -49,7 +50,7 @@
class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvaluator {
- private final PythonLibraryEvaluator libraryEvaluator;
+ private final ILibraryEvaluator libraryEvaluator;
private final ArrayBackedValueStorage resultBuffer = new ArrayBackedValueStorage();
private final ByteBuffer argHolder;
@@ -115,7 +116,7 @@
return;
}
try {
- ByteBuffer res = libraryEvaluator.callPython(fnId, argTypes, argValues, nullCall);
+ ByteBuffer res = libraryEvaluator.call(fnId, argTypes, argValues, nullCall);
resultBuffer.reset();
wrap(res, resultBuffer.getDataOutput());
} catch (Exception e) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
new file mode 100644
index 0000000..056aa9a
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.library;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.lang.invoke.VarHandle;
+import java.net.ProtocolFamily;
+import java.net.SocketAddress;
+import java.net.StandardProtocolFamily;
+import java.nio.channels.Channels;
+import java.nio.channels.SocketChannel;
+import java.nio.file.Path;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.external.ipc.PythonDomainSocketProto;
+import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.TaskAttemptId;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class PythonLibraryDomainSocketEvaluator extends AbstractLibrarySocketEvaluator {
+
+ private final ILibraryManager libMgr;
+ private final Path sockPath;
+ SocketChannel chan;
+ ProcessHandle pid;
+ private static final Logger LOGGER = LogManager.getLogger(ExternalLibraryManager.class);
+
+ public PythonLibraryDomainSocketEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId, ILibraryManager libMgr,
+ TaskAttemptId task, IWarningCollector warningCollector, SourceLocation sourceLoc, Path sockPath) {
+ super(jobId, evaluatorId, task, warningCollector, sourceLoc);
+ this.libMgr = libMgr;
+ this.sockPath = sockPath;
+ }
+
+ public void start() throws IOException, AsterixException {
+ PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
+ PythonLibrary library =
+ (PythonLibrary) libMgr.getLibrary(fnId.getLibraryDataverseName(), fnId.getLibraryName());
+ String wd = library.getFile().getAbsolutePath();
+ MethodHandles.Lookup lookup = MethodHandles.lookup();
+ SocketAddress sockAddr;
+ try {
+ VarHandle sockEnum = lookup.in(StandardProtocolFamily.class)
+ .findStaticVarHandle(StandardProtocolFamily.class, "UNIX", StandardProtocolFamily.class);
+ Class domainSock = Class.forName("java.net.UnixDomainSocketAddress");
+ MethodType unixDomainSockAddrType = MethodType.methodType(domainSock, Path.class);
+ MethodHandle unixDomainSockAddr = lookup.findStatic(domainSock, "of", unixDomainSockAddrType);
+ MethodType sockChanMethodType = MethodType.methodType(SocketChannel.class, ProtocolFamily.class);
+ MethodHandle sockChanOpen = lookup.findStatic(SocketChannel.class, "open", sockChanMethodType);
+ sockAddr = ((SocketAddress) unixDomainSockAddr.invoke(sockPath));
+ chan = (SocketChannel) sockChanOpen.invoke(sockEnum.get());
+ } catch (Throwable e) {
+ throw HyracksDataException.create(ErrorCode.LOCAL_NETWORK_ERROR, e);
+ }
+ chan.connect(sockAddr);
+ proto = new PythonDomainSocketProto(Channels.newOutputStream(chan), chan, wd);
+ proto.start();
+ proto.helo();
+ this.pid = ((PythonDomainSocketProto) proto).getPid();
+ }
+
+ @Override
+ public void deallocate() {
+ try {
+ if (proto != null) {
+ proto.quit();
+ }
+ if (chan != null) {
+ chan.close();
+ }
+ } catch (IOException e) {
+ LOGGER.error("Caught exception exiting Python UDF:", e);
+ }
+ if (pid != null && pid.isAlive()) {
+ LOGGER.error("Python UDF " + pid.pid() + " did not exit as expected.");
+ }
+ }
+
+ static PythonLibraryDomainSocketEvaluator getInstance(IExternalFunctionInfo finfo, ILibraryManager libMgr,
+ IHyracksTaskContext ctx, IWarningCollector warningCollector, SourceLocation sourceLoc)
+ throws IOException, AsterixException {
+ PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
+ finfo.getLibraryName(), Thread.currentThread());
+ PythonLibraryDomainSocketEvaluator evaluator =
+ (PythonLibraryDomainSocketEvaluator) ctx.getStateObject(evaluatorId);
+ if (evaluator == null) {
+ Path sockPath = Path.of(ctx.getJobletContext().getServiceContext().getAppConfig()
+ .getString(NCConfig.Option.PYTHON_DS_PATH));
+ evaluator = new PythonLibraryDomainSocketEvaluator(ctx.getJobletContext().getJobId(), evaluatorId, libMgr,
+ ctx.getTaskAttemptId(), warningCollector, sourceLoc, sockPath);
+ ctx.getJobletContext().registerDeallocatable(evaluator);
+ evaluator.start();
+ ctx.setStateObject(evaluator);
+ }
+ return evaluator;
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
deleted file mode 100644
index f82b30d..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library;
-
-import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_UDF_EXCEPTION;
-import static org.msgpack.core.MessagePack.Code.ARRAY16;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
-import org.apache.asterix.external.ipc.PythonIPCProto;
-import org.apache.asterix.external.library.msgpack.MessagePackUtils;
-import org.apache.asterix.om.functions.IExternalFunctionInfo;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.TypeTagUtil;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.TaskAttemptId;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.resources.IDeallocatable;
-import org.apache.hyracks.data.std.api.IValueReference;
-import org.apache.hyracks.data.std.primitive.TaggedValuePointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.std.base.AbstractStateObject;
-import org.apache.hyracks.ipc.impl.IPCSystem;
-
-public class PythonLibraryEvaluator extends AbstractStateObject implements IDeallocatable {
-
- public static final String ENTRYPOINT = "entrypoint.py";
- public static final String SITE_PACKAGES = "site-packages";
-
- private Process p;
- private ILibraryManager libMgr;
- private File pythonHome;
- private PythonIPCProto proto;
- private ExternalFunctionResultRouter router;
- private IPCSystem ipcSys;
- private String sitePkgs;
- private List<String> pythonArgs;
- private Map<String, String> pythonEnv;
- private TaskAttemptId task;
- private IWarningCollector warningCollector;
- private SourceLocation sourceLoc;
-
- public PythonLibraryEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId, ILibraryManager libMgr,
- File pythonHome, String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv,
- ExternalFunctionResultRouter router, IPCSystem ipcSys, TaskAttemptId task,
- IWarningCollector warningCollector, SourceLocation sourceLoc) {
- super(jobId, evaluatorId);
- this.libMgr = libMgr;
- this.pythonHome = pythonHome;
- this.sitePkgs = sitePkgs;
- this.pythonArgs = pythonArgs;
- this.pythonEnv = pythonEnv;
- this.router = router;
- this.task = task;
- this.ipcSys = ipcSys;
- this.warningCollector = warningCollector;
- this.sourceLoc = sourceLoc;
- }
-
- private void initialize() throws IOException, AsterixException {
- PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
- PythonLibrary library =
- (PythonLibrary) libMgr.getLibrary(fnId.getLibraryDataverseName(), fnId.getLibraryName());
- String wd = library.getFile().getAbsolutePath();
- int port = ipcSys.getSocketAddress().getPort();
- List<String> args = new ArrayList<>();
- args.add(pythonHome.getAbsolutePath());
- args.addAll(pythonArgs);
- args.add(ENTRYPOINT);
- args.add(InetAddress.getLoopbackAddress().getHostAddress());
- args.add(Integer.toString(port));
- args.add(sitePkgs);
- ProcessBuilder pb = new ProcessBuilder(args.toArray(new String[0]));
- pb.environment().putAll(pythonEnv);
- pb.directory(new File(wd));
- p = pb.start();
- proto = new PythonIPCProto(p.getOutputStream(), router, p);
- proto.start();
- proto.helo();
- }
-
- public long initialize(IExternalFunctionInfo finfo) throws IOException, AsterixException {
- List<String> externalIdents = finfo.getExternalIdentifier();
- String packageModule = externalIdents.get(0);
- String clazz;
- String fn;
- String externalIdent1 = externalIdents.get(1);
- int idx = externalIdent1.lastIndexOf('.');
- if (idx >= 0) {
- clazz = externalIdent1.substring(0, idx);
- fn = externalIdent1.substring(idx + 1);
- } else {
- clazz = null;
- fn = externalIdent1;
- }
- return proto.init(packageModule, clazz, fn);
- }
-
- public ByteBuffer callPython(long id, IAType[] argTypes, IValueReference[] valueReferences, boolean nullCall)
- throws IOException {
- ByteBuffer ret = null;
- try {
- ret = proto.call(id, argTypes, valueReferences, nullCall);
- } catch (AsterixException e) {
- if (warningCollector.shouldWarn()) {
- warningCollector.warn(Warning.of(sourceLoc, EXTERNAL_UDF_EXCEPTION, e.getMessage()));
- }
- }
- return ret;
- }
-
- public ByteBuffer callPythonMulti(long id, ArrayBackedValueStorage arguments, int numTuples) throws IOException {
- ByteBuffer ret = null;
- try {
- ret = proto.callMulti(id, arguments, numTuples);
- } catch (AsterixException e) {
- if (warningCollector.shouldWarn()) {
- warningCollector.warn(Warning.of(sourceLoc, EXTERNAL_UDF_EXCEPTION, e.getMessage()));
- }
- }
- return ret;
- }
-
- @Override
- public void deallocate() {
- if (p != null) {
- boolean dead = false;
- try {
- p.destroy();
- dead = p.waitFor(100, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- //gonna kill it anyway
- }
- if (!dead) {
- p.destroyForcibly();
- }
- }
- router.removeRoute(proto.getRouteId());
- }
-
- public static ATypeTag peekArgument(IAType type, IValueReference valueReference) throws HyracksDataException {
- ATypeTag tag = type.getTypeTag();
- if (tag == ATypeTag.ANY) {
- TaggedValuePointable pointy = TaggedValuePointable.FACTORY.createPointable();
- pointy.set(valueReference);
- ATypeTag rtTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(pointy.getTag());
- IAType rtType = TypeTagUtil.getBuiltinTypeByTag(rtTypeTag);
- return MessagePackUtils.peekUnknown(rtType);
- } else {
- return MessagePackUtils.peekUnknown(type);
- }
- }
-
- public static void setVoidArgument(ArrayBackedValueStorage argHolder) throws IOException {
- argHolder.getDataOutput().writeByte(ARRAY16);
- argHolder.getDataOutput().writeShort((short) 0);
- }
-
- public static PythonLibraryEvaluator getInstance(IExternalFunctionInfo finfo, ILibraryManager libMgr,
- ExternalFunctionResultRouter router, IPCSystem ipcSys, File pythonHome, IHyracksTaskContext ctx,
- String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv, IWarningCollector warningCollector,
- SourceLocation sourceLoc) throws IOException, AsterixException {
- PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
- finfo.getLibraryName(), Thread.currentThread());
- PythonLibraryEvaluator evaluator = (PythonLibraryEvaluator) ctx.getStateObject(evaluatorId);
- if (evaluator == null) {
- evaluator = new PythonLibraryEvaluator(ctx.getJobletContext().getJobId(), evaluatorId, libMgr, pythonHome,
- sitePkgs, pythonArgs, pythonEnv, router, ipcSys, ctx.getTaskAttemptId(), warningCollector,
- sourceLoc);
- ctx.getJobletContext().registerDeallocatable(evaluator);
- evaluator.initialize();
- ctx.setStateObject(evaluator);
- }
- return evaluator;
- }
-
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
index 06c9bc9..63a6ec3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
@@ -18,10 +18,12 @@
*/
package org.apache.asterix.external.library;
-import static org.apache.asterix.external.library.PythonLibraryEvaluator.SITE_PACKAGES;
+import static org.apache.asterix.external.library.PythonLibraryTCPSocketEvaluator.SITE_PACKAGES;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -31,8 +33,10 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.external.api.ILibraryEvaluator;
import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.hyracks.api.config.IApplicationConfig;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -40,83 +44,116 @@
import org.apache.hyracks.ipc.impl.IPCSystem;
public class PythonLibraryEvaluatorFactory {
- private final ILibraryManager libraryManager;
- private final IPCSystem ipcSys;
- private final File pythonPath;
- private final IHyracksTaskContext ctx;
- private final ExternalFunctionResultRouter router;
- private final String sitePackagesPath;
- private final List<String> pythonArgs;
- private final Map<String, String> pythonEnv;
+
+ private ILibraryManager libraryManager;
+ private IPCSystem ipcSys;
+ private File pythonPath;
+ private IHyracksTaskContext ctx;
+ private ExternalFunctionResultRouter router;
+ private String sitePackagesPath;
+ private List<String> pythonArgs;
+ private Map<String, String> pythonEnv;
+
+ private boolean domainSockEnable;
public PythonLibraryEvaluatorFactory(IHyracksTaskContext ctx) throws AsterixException {
this.ctx = ctx;
+ String dsPath =
+ ctx.getJobletContext().getServiceContext().getAppConfig().getString(NCConfig.Option.PYTHON_DS_PATH);
+ config(dsPath == null ? null : Path.of(dsPath));
libraryManager = ((INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext())
.getLibraryManager();
- router = libraryManager.getRouter();
- ipcSys = libraryManager.getIPCI();
- IApplicationConfig appCfg = ctx.getJobletContext().getServiceContext().getAppConfig();
- String pythonPathCmd = appCfg.getString(NCConfig.Option.PYTHON_CMD);
- boolean findPython = appCfg.getBoolean(NCConfig.Option.PYTHON_CMD_AUTOLOCATE);
- pythonArgs = new ArrayList<>();
- if (pythonPathCmd == null) {
- if (findPython) {
- //if absolute path to interpreter is not specified, try to use environmental python
- pythonPathCmd = "/usr/bin/env";
- pythonArgs.add("python3");
- } else {
- throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION, "Python interpreter not specified, and "
- + NCConfig.Option.PYTHON_CMD_AUTOLOCATE.ini() + " is false");
- }
- }
- pythonEnv = new HashMap<>();
- String[] envRaw = appCfg.getStringArray((NCConfig.Option.PYTHON_ENV));
- if (envRaw != null) {
- for (String rawEnvArg : envRaw) {
- //TODO: i think equals is shared among all unixes and windows. but it needs verification
- if (rawEnvArg.length() < 1) {
- continue;
- }
- String[] rawArgSplit = rawEnvArg.split("(?<!\\\\)=", 2);
- if (rawArgSplit.length < 2) {
+ if (!domainSockEnable) {
+ router = libraryManager.getRouter();
+ ipcSys = libraryManager.getIPCI();
+ IApplicationConfig appCfg = ctx.getJobletContext().getServiceContext().getAppConfig();
+ String pythonPathCmd = appCfg.getString(NCConfig.Option.PYTHON_CMD);
+ boolean findPython = appCfg.getBoolean(NCConfig.Option.PYTHON_CMD_AUTOLOCATE);
+ pythonArgs = new ArrayList<>();
+ if (pythonPathCmd == null) {
+ if (findPython) {
+ //if absolute path to interpreter is not specified, try to use environmental python
+ pythonPathCmd = "/usr/bin/env";
+ pythonArgs.add("python3");
+ } else {
throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
- "Invalid environment variable format detected.");
- }
- pythonEnv.put(rawArgSplit[0], rawArgSplit[1]);
- }
- }
- pythonPath = new File(pythonPathCmd);
- List<String> sitePkgs = new ArrayList<>();
- sitePkgs.add(SITE_PACKAGES);
- String[] addlSitePackages = appCfg.getStringArray((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES));
- for (String sitePkg : addlSitePackages) {
- if (sitePkg.length() > 0) {
- sitePkgs.add(sitePkg);
- }
- }
- if (appCfg.getBoolean(NCConfig.Option.PYTHON_USE_BUNDLED_MSGPACK)) {
- sitePkgs.add("ipc" + File.separator + SITE_PACKAGES + File.separator);
- }
- String[] pythonArgsRaw = appCfg.getStringArray(NCConfig.Option.PYTHON_ARGS);
- if (pythonArgsRaw != null) {
- for (String arg : pythonArgsRaw) {
- if (arg.length() > 0) {
- pythonArgs.add(arg);
+ "Python interpreter not specified or domain socket not found, and "
+ + NCConfig.Option.PYTHON_CMD_AUTOLOCATE.ini() + " is false");
}
}
+ pythonEnv = new HashMap<>();
+ String[] envRaw = appCfg.getStringArray((NCConfig.Option.PYTHON_ENV));
+ if (envRaw != null) {
+ for (String rawEnvArg : envRaw) {
+ //TODO: i think equals is shared among all unixes and windows. but it needs verification
+ if (rawEnvArg.length() < 1) {
+ continue;
+ }
+ String[] rawArgSplit = rawEnvArg.split("(?<!\\\\)=", 2);
+ if (rawArgSplit.length < 2) {
+ throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ "Invalid environment variable format detected.");
+ }
+ pythonEnv.put(rawArgSplit[0], rawArgSplit[1]);
+ }
+ }
+ pythonPath = new File(pythonPathCmd);
+ List<String> sitePkgs = new ArrayList<>();
+ sitePkgs.add(SITE_PACKAGES);
+ String[] addlSitePackages = appCfg.getStringArray((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES));
+ for (String sitePkg : addlSitePackages) {
+ if (sitePkg.length() > 0) {
+ sitePkgs.add(sitePkg);
+ }
+ }
+ if (appCfg.getBoolean(NCConfig.Option.PYTHON_USE_BUNDLED_MSGPACK)) {
+ sitePkgs.add("ipc" + File.separator + SITE_PACKAGES + File.separator);
+ }
+ String[] pythonArgsRaw = appCfg.getStringArray(NCConfig.Option.PYTHON_ARGS);
+ if (pythonArgsRaw != null) {
+ for (String arg : pythonArgsRaw) {
+ if (arg.length() > 0) {
+ pythonArgs.add(arg);
+ }
+ }
+ }
+ StringBuilder sitePackagesPathBuilder = new StringBuilder();
+ for (int i = 0; i < sitePkgs.size() - 1; i++) {
+ sitePackagesPathBuilder.append(sitePkgs.get(i));
+ sitePackagesPathBuilder.append(File.pathSeparator);
+ }
+ sitePackagesPathBuilder.append(sitePkgs.get(sitePkgs.size() - 1));
+ sitePackagesPath = sitePackagesPathBuilder.toString();
}
- StringBuilder sitePackagesPathBuilder = new StringBuilder();
- for (int i = 0; i < sitePkgs.size() - 1; i++) {
- sitePackagesPathBuilder.append(sitePkgs.get(i));
- sitePackagesPathBuilder.append(File.pathSeparator);
- }
- sitePackagesPathBuilder.append(sitePkgs.get(sitePkgs.size() - 1));
- sitePackagesPath = sitePackagesPathBuilder.toString();
}
- public PythonLibraryEvaluator getEvaluator(IExternalFunctionInfo fnInfo, SourceLocation sourceLoc)
+ public ILibraryEvaluator getEvaluator(IExternalFunctionInfo fnInfo, SourceLocation sourceLoc)
throws IOException, AsterixException {
- return PythonLibraryEvaluator.getInstance(fnInfo, libraryManager, router, ipcSys, pythonPath, ctx,
- sitePackagesPath, pythonArgs, pythonEnv, ctx.getWarningCollector(), sourceLoc);
+ if (domainSockEnable) {
+ return PythonLibraryDomainSocketEvaluator.getInstance(fnInfo, libraryManager, ctx,
+ ctx.getWarningCollector(), sourceLoc);
+ } else {
+ return PythonLibraryTCPSocketEvaluator.getInstance(fnInfo, libraryManager, router, ipcSys, pythonPath, ctx,
+ sitePackagesPath, pythonArgs, pythonEnv, ctx.getWarningCollector(), sourceLoc);
+ }
+ }
+
+ private void config(Path sockPath) throws AsterixException {
+ if (sockPath == null) {
+ domainSockEnable = false;
+ return;
+ }
+ Runtime rt = Runtime.getRuntime();
+ if (rt.version().feature() >= 17 && SystemUtils.IS_OS_LINUX) {
+ if (Files.exists(sockPath)) {
+ domainSockEnable = true;
+ } else {
+ throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ "Domain socket was not found at specified path");
+ }
+ } else {
+ throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ "Domain socket path specified, but Java version is below 17 or OS is not Linux");
+ }
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
new file mode 100644
index 0000000..385d738
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.library;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
+import org.apache.asterix.external.ipc.PythonTCPSocketProto;
+import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.TaskAttemptId;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.ipc.impl.IPCSystem;
+
+public class PythonLibraryTCPSocketEvaluator extends AbstractLibrarySocketEvaluator {
+
+ public static final String ENTRYPOINT = "entrypoint.py";
+ public static final String SITE_PACKAGES = "site-packages";
+
+ private Process p;
+ private ILibraryManager libMgr;
+ private File pythonHome;
+ private ExternalFunctionResultRouter router;
+ private IPCSystem ipcSys;
+ private String sitePkgs;
+ private List<String> pythonArgs;
+ private Map<String, String> pythonEnv;
+
+ public PythonLibraryTCPSocketEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId, ILibraryManager libMgr,
+ File pythonHome, String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv,
+ ExternalFunctionResultRouter router, IPCSystem ipcSys, TaskAttemptId task,
+ IWarningCollector warningCollector, SourceLocation sourceLoc) {
+ super(jobId, evaluatorId, task, warningCollector, sourceLoc);
+ this.libMgr = libMgr;
+ this.pythonHome = pythonHome;
+ this.sitePkgs = sitePkgs;
+ this.pythonArgs = pythonArgs;
+ this.pythonEnv = pythonEnv;
+ this.router = router;
+ this.ipcSys = ipcSys;
+ }
+
+ @Override
+ public void start() throws IOException, AsterixException {
+ PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
+ PythonLibrary library =
+ (PythonLibrary) libMgr.getLibrary(fnId.getLibraryDataverseName(), fnId.getLibraryName());
+ String wd = library.getFile().getAbsolutePath();
+ int port = ipcSys.getSocketAddress().getPort();
+ List<String> args = new ArrayList<>();
+ args.add(pythonHome.getAbsolutePath());
+ args.addAll(pythonArgs);
+ args.add(ENTRYPOINT);
+ args.add(InetAddress.getLoopbackAddress().getHostAddress());
+ args.add(Integer.toString(port));
+ args.add(sitePkgs);
+ ProcessBuilder pb = new ProcessBuilder(args.toArray(new String[0]));
+ pb.environment().putAll(pythonEnv);
+ pb.directory(new File(wd));
+ p = pb.start();
+ proto = new PythonTCPSocketProto(p.getOutputStream(), router, p);
+ proto.start();
+ proto.helo();
+ }
+
+ @Override
+ public void deallocate() {
+ if (p != null) {
+ boolean dead = false;
+ try {
+ p.destroy();
+ dead = p.waitFor(100, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ //gonna kill it anyway
+ }
+ if (!dead) {
+ p.destroyForcibly();
+ }
+ }
+ router.removeRoute(proto.getRouteId());
+ }
+
+ static PythonLibraryTCPSocketEvaluator getInstance(IExternalFunctionInfo finfo, ILibraryManager libMgr,
+ ExternalFunctionResultRouter router, IPCSystem ipcSys, File pythonHome, IHyracksTaskContext ctx,
+ String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv, IWarningCollector warningCollector,
+ SourceLocation sourceLoc) throws IOException, AsterixException {
+ PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
+ finfo.getLibraryName(), Thread.currentThread());
+ PythonLibraryTCPSocketEvaluator evaluator = (PythonLibraryTCPSocketEvaluator) ctx.getStateObject(evaluatorId);
+ if (evaluator == null) {
+ evaluator = new PythonLibraryTCPSocketEvaluator(ctx.getJobletContext().getJobId(), evaluatorId, libMgr,
+ pythonHome, sitePkgs, pythonArgs, pythonEnv, router, ipcSys, ctx.getTaskAttemptId(),
+ warningCollector, sourceLoc);
+ ctx.getJobletContext().registerDeallocatable(evaluator);
+ evaluator.start();
+ ctx.setStateObject(evaluator);
+ }
+ return evaluator;
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java
index a90a183..6efbb6e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java
@@ -124,7 +124,7 @@
int s = pointable.getStartOffset();
int i = AInt32SerializerDeserializer.getInt(b, s + 1);
out.writeByte(INT32);
- out.writeByte(i);
+ out.writeInt(i);
return null;
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
index 741dad2..5f8a3f0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
@@ -33,12 +33,13 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.external.ipc.PythonIPCProto;
-import org.apache.asterix.external.library.PythonLibraryEvaluator;
+import org.apache.asterix.external.api.IExternalLangIPCProto;
+import org.apache.asterix.external.api.ILibraryEvaluator;
import org.apache.asterix.external.library.PythonLibraryEvaluatorFactory;
import org.apache.asterix.external.library.msgpack.MessageUnpackerToADM;
import org.apache.asterix.external.library.msgpack.MsgPackPointableVisitor;
import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.om.functions.IExternalFunctionDescriptor;
import org.apache.asterix.om.pointables.PointableAllocator;
import org.apache.asterix.om.types.ATypeTag;
@@ -50,6 +51,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.data.std.primitive.TaggedValuePointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -87,7 +89,7 @@
private ArrayBackedValueStorage outputWrapper;
private List<ArrayBackedValueStorage> argHolders;
ArrayTupleBuilder tupleBuilder;
- private List<Pair<Long, PythonLibraryEvaluator>> libraryEvaluators;
+ private List<Pair<Long, ILibraryEvaluator>> libraryEvaluators;
private ATypeTag[][] nullCalls;
private int[] numCalls;
private VoidPointable ref;
@@ -97,6 +99,7 @@
private MessageUnpackerToADM unpackerToADM;
private PointableAllocator pointableAllocator;
private MsgPackPointableVisitor pointableVisitor;
+ private TaggedValuePointable anyPointer;
@Override
public void open() throws HyracksDataException {
@@ -109,7 +112,7 @@
try {
PythonLibraryEvaluatorFactory evalFactory = new PythonLibraryEvaluatorFactory(ctx);
for (IExternalFunctionDescriptor fnDesc : fnDescs) {
- PythonLibraryEvaluator eval = evalFactory.getEvaluator(fnDesc.getFunctionInfo(), sourceLoc);
+ ILibraryEvaluator eval = evalFactory.getEvaluator(fnDesc.getFunctionInfo(), sourceLoc);
long id = eval.initialize(fnDesc.getFunctionInfo());
libraryEvaluators.add(new Pair<>(id, eval));
}
@@ -133,6 +136,7 @@
unpackerToADM = new MessageUnpackerToADM();
pointableAllocator = new PointableAllocator();
pointableVisitor = new MsgPackPointableVisitor();
+ anyPointer = TaggedValuePointable.FACTORY.createPointable();
}
private void resetBuffers(int numTuples, int[] numCalls) {
@@ -177,8 +181,12 @@
int numEntries = unpacker.unpackArrayHeader();
for (int j = 0; j < numEntries; j++) {
if (ctx.getWarningCollector().shouldWarn()) {
- ctx.getWarningCollector().warn(Warning.of(sourceLoc,
- ErrorCode.EXTERNAL_UDF_EXCEPTION, unpacker.unpackString()));
+ //TODO: in domain socket mode, a NUL can appear at the end of the stacktrace strings.
+ // this should probably not happen but warnings with control characters should
+ // also be properly escaped
+ ctx.getWarningCollector()
+ .warn(Warning.of(sourceLoc, ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ unpacker.unpackString().replace('\0', ' ')));
}
}
} catch (MessagePackException e) {
@@ -211,8 +219,8 @@
for (int colIdx = 0; colIdx < cols.length; colIdx++) {
ref.set(buffer.array(), tRef.getFieldStart(cols[colIdx]),
tRef.getFieldLength(cols[colIdx]));
- ATypeTag argumentPresence = PythonLibraryEvaluator
- .peekArgument(fnDescs[func].getArgumentTypes()[colIdx], ref);
+ ATypeTag argumentPresence = ExternalDataUtils
+ .peekArgument(fnDescs[func].getArgumentTypes()[colIdx], ref, anyPointer);
argumentStatus = handleNullMatrix(func, t, argumentPresence, argumentStatus);
}
}
@@ -224,7 +232,7 @@
for (int colIdx = 0; colIdx < cols.length; colIdx++) {
ref.set(buffer.array(), tRef.getFieldStart(cols[colIdx]),
tRef.getFieldLength(cols[colIdx]));
- PythonIPCProto.visitValueRef(fnDescs[func].getArgumentTypes()[colIdx],
+ IExternalLangIPCProto.visitValueRef(fnDescs[func].getArgumentTypes()[colIdx],
argHolders.get(func).getDataOutput(), ref, pointableAllocator,
pointableVisitor, fnDescs[func].getFunctionInfo().getNullCall());
}
@@ -232,21 +240,25 @@
numCalls[func]--;
}
if (cols.length == 0) {
- PythonLibraryEvaluator.setVoidArgument(argHolders.get(func));
+ ExternalDataUtils.setVoidArgument(argHolders.get(func));
}
}
}
//TODO: maybe this could be done in parallel for each unique library evaluator?
for (int argHolderIdx = 0; argHolderIdx < argHolders.size(); argHolderIdx++) {
- Pair<Long, PythonLibraryEvaluator> fnEval = libraryEvaluators.get(argHolderIdx);
- ByteBuffer columnResult = fnEval.getSecond().callPythonMulti(fnEval.getFirst(),
+ Pair<Long, ILibraryEvaluator> fnEval = libraryEvaluators.get(argHolderIdx);
+ ByteBuffer columnResult = fnEval.getSecond().callMulti(fnEval.getFirst(),
argHolders.get(argHolderIdx), numCalls[argHolderIdx]);
if (columnResult != null) {
Pair<ByteBuffer, Counter> resultholder = batchResults.get(argHolderIdx);
- if (resultholder.getFirst().capacity() < columnResult.capacity()) {
- ByteBuffer realloc = ctx.reallocateFrame(resultholder.getFirst(),
- columnResult.capacity() * 2, false);
+ if (resultholder.getFirst().capacity() < columnResult.remaining()) {
+ ByteBuffer realloc =
+ ctx.reallocateFrame(resultholder.getFirst(),
+ ctx.getInitialFrameSize()
+ * ((columnResult.remaining() / ctx.getInitialFrameSize()) + 1),
+ false);
+ realloc.limit(columnResult.limit());
resultholder.setFirst(realloc);
}
ByteBuffer resultBuf = resultholder.getFirst();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java
index 82b8113..0250ba8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.api.job.profiling.IOperatorStats;
+import org.apache.hyracks.api.job.profiling.NoOpOperatorStats;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
import org.apache.hyracks.storage.am.common.api.ITupleFilter;
@@ -66,23 +67,18 @@
return new AbstractUnaryOutputSourceOperatorNodePushable() {
- private IOperatorStats stats;
+ private IOperatorStats stats = NoOpOperatorStats.INSTANCE;
@Override
public void initialize() throws HyracksDataException {
IDataSourceAdapter adapter;
- if (ctx.getStatsCollector() != null) {
- stats = ctx.getStatsCollector().getOrAddOperatorStats(getDisplayName());
- }
try {
writer.open();
ITupleFilter tupleFilter =
tupleFilterFactory != null ? tupleFilterFactory.createTupleFilter(ctx) : null;
adapter = adapterFactory.createAdapter(ctx, partition);
adapter.start(partition, writer, tupleFilter, outputLimit);
- if (stats != null) {
- stats.getTupleCounter().update(adapter.getProcessedTuples());
- }
+ stats.getInputTupleCounter().update(adapter.getProcessedTuples());
} catch (Exception e) {
writer.fail();
throw HyracksDataException.create(e);
@@ -90,6 +86,11 @@
writer.close();
}
}
+
+ @Override
+ public void setOperatorStats(IOperatorStats stats) {
+ this.stats = stats;
+ }
};
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 050a080..479679e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -334,108 +334,4 @@
*/
public static final Set<String> VALID_TIME_ZONES = Set.of(TimeZone.getAvailableIDs());
}
-
- public static class AwsS3 {
- private AwsS3() {
- throw new AssertionError("do not instantiate");
- }
-
- public static final String REGION_FIELD_NAME = "region";
- public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
- public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
- public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
- public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
-
- // AWS S3 specific error codes
- public static final String ERROR_INTERNAL_ERROR = "InternalError";
- public static final String ERROR_SLOW_DOWN = "SlowDown";
- public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
-
- public static boolean isRetryableError(String errorCode) {
- return errorCode.equals(ERROR_INTERNAL_ERROR) || errorCode.equals(ERROR_SLOW_DOWN);
- }
-
- /*
- * Hadoop-AWS
- * AWS connectors for s3 and s3n are deprecated.
- */
- public static final String HADOOP_ACCESS_KEY_ID = "fs.s3a.access.key";
- public static final String HADOOP_SECRET_ACCESS_KEY = "fs.s3a.secret.key";
- public static final String HADOOP_SESSION_TOKEN = "fs.s3a.session.token";
- public static final String HADOOP_REGION = "fs.s3a.region";
- public static final String HADOOP_SERVICE_END_POINT = "fs.s3a.endpoint";
-
- /*
- * Internal configurations
- */
- //Allows accessing directories as file system path
- public static final String HADOOP_PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
- //The number of maximum HTTP connections in connection pool
- public static final String HADOOP_S3_CONNECTION_POOL_SIZE = "fs.s3a.connection.maximum";
- //S3 used protocol
- public static final String HADOOP_S3_PROTOCOL = "s3a";
-
- //Hadoop credentials provider key
- public static final String HADOOP_CREDENTIAL_PROVIDER_KEY = "fs.s3a.aws.credentials.provider";
- //Anonymous credential provider
- public static final String HADOOP_ANONYMOUS_ACCESS = "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider";
- //Temporary credential provider
- public static final String HADOOP_TEMP_ACCESS = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
-
- }
-
- /*
- * Note: Azure Blob and Azure Datalake use identical authentication, so they are using the same properties.
- * If they end up diverging, then properties for AzureBlob and AzureDataLake need to be created.
- */
- public static class Azure {
- private Azure() {
- throw new AssertionError("do not instantiate");
- }
-
- /*
- * Asterix Configuration Keys
- */
- public static final String MANAGED_IDENTITY_ID_FIELD_NAME = "managedIdentityId";
- public static final String ACCOUNT_NAME_FIELD_NAME = "accountName";
- public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey";
- public static final String SHARED_ACCESS_SIGNATURE_FIELD_NAME = "sharedAccessSignature";
- public static final String TENANT_ID_FIELD_NAME = "tenantId";
- public static final String CLIENT_ID_FIELD_NAME = "clientId";
- public static final String CLIENT_SECRET_FIELD_NAME = "clientSecret";
- public static final String CLIENT_CERTIFICATE_FIELD_NAME = "clientCertificate";
- public static final String CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME = "clientCertificatePassword";
- public static final String ENDPOINT_FIELD_NAME = "endpoint";
-
- // Specific Azure data lake property
- /*
- The behavior of Data Lake (true file system) is to read the files of the specified prefix only, example:
- storage/myData/personal/file1.json
- storage/myData/personal/file2.json
- storage/myData/file3.json
- If the prefix used is "myData", then only the file file3.json is read. However, if the property "recursive"
- is set to "true" when creating the external dataset, then it goes recursively overall the paths, and the result
- is file1.json, file2.json and file3.json.
- */
- public static final String RECURSIVE_FIELD_NAME = "recursive";
-
- /*
- * Hadoop-Azure
- */
- //Used when accountName and accessKey are provided
- public static final String HADOOP_AZURE_FS_ACCOUNT_KEY = "fs.azure.account.key";
- //Used when a connectionString is provided
- public static final String HADOOP_AZURE_FS_SAS = "fs.azure.sas";
- public static final String HADOOP_AZURE_BLOB_PROTOCOL = "wasbs";
- public static final String HADOOP_AZURE_DATALAKE_PROTOCOL = "abfss";
- }
-
- public static class GCS {
- private GCS() {
- throw new AssertionError("do not instantiate");
- }
-
- public static final String APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME = "applicationDefaultCredentials";
- public static final String JSON_CREDENTIALS_FIELD_NAME = "jsonCredentials";
- }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 29e04e9..1605360 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,67 +18,23 @@
*/
package org.apache.asterix.external.util;
-import static com.google.cloud.storage.Storage.BlobListOption;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
-import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE;
-import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_PARAM_VAL;
-import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME;
-import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_REQUIRED;
-import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
-import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
-import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT;
-import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_ACCESS_KEY_ID;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_ANONYMOUS_ACCESS;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_CREDENTIAL_PROVIDER_KEY;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_PATH_STYLE_ACCESS;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_S3_CONNECTION_POOL_SIZE;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_S3_PROTOCOL;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_SECRET_ACCESS_KEY;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_SESSION_TOKEN;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_TEMP_ACCESS;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.ACCOUNT_KEY_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.ACCOUNT_NAME_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_CERTIFICATE_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_SECRET_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.ENDPOINT_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.HADOOP_AZURE_BLOB_PROTOCOL;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.HADOOP_AZURE_FS_ACCOUNT_KEY;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.HADOOP_AZURE_FS_SAS;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.MANAGED_IDENTITY_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.RECURSIVE_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.Azure.TENANT_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.GCS.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.GCS.JSON_CREDENTIALS_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ADAPTER_NAME_GCS;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXCLUDE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
-import static org.apache.asterix.external.util.ExternalDataConstants.KEY_FORMAT;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_INCLUDE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.validateAzureBlobProperties;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.validateAzureDataLakeProperties;
+import static org.apache.asterix.external.util.google.gcs.GCSUtils.validateProperties;
import static org.apache.asterix.runtime.evaluators.functions.StringEvaluatorUtils.RESERVED_REGEX_CHARS;
-import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
+import static org.msgpack.core.MessagePack.Code.ARRAY16;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
@@ -87,7 +43,6 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.function.BiPredicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -107,21 +62,25 @@
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory.IncludeExcludeMatcher;
import org.apache.asterix.external.library.JavaLibrary;
+import org.apache.asterix.external.library.msgpack.MessagePackUtils;
import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
+import org.apache.asterix.external.util.aws.s3.S3Utils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeTagUtil;
import org.apache.asterix.runtime.evaluators.common.NumberUtils;
-import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
import org.apache.asterix.runtime.projection.FunctionCallInformation;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.TaggedValuePointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.parsers.BooleanParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.DoubleParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.FloatParserFactory;
@@ -131,46 +90,6 @@
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import org.apache.hyracks.util.StorageUtil;
-import com.azure.core.credential.AzureSasCredential;
-import com.azure.core.http.rest.PagedIterable;
-import com.azure.identity.ClientCertificateCredentialBuilder;
-import com.azure.identity.ClientSecretCredentialBuilder;
-import com.azure.identity.ManagedIdentityCredentialBuilder;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.models.BlobItem;
-import com.azure.storage.blob.models.ListBlobsOptions;
-import com.azure.storage.common.StorageSharedKeyCredential;
-import com.azure.storage.common.policy.RequestRetryOptions;
-import com.azure.storage.file.datalake.DataLakeFileSystemClient;
-import com.azure.storage.file.datalake.DataLakeServiceClient;
-import com.azure.storage.file.datalake.DataLakeServiceClientBuilder;
-import com.azure.storage.file.datalake.models.ListPathsOptions;
-import com.azure.storage.file.datalake.models.PathItem;
-import com.google.api.gax.paging.Page;
-import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.storage.Blob;
-import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.StorageOptions;
-
-import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.exception.SdkException;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.S3ClientBuilder;
-import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
-import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
-import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
-import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
-import software.amazon.awssdk.services.s3.model.S3Exception;
-import software.amazon.awssdk.services.s3.model.S3Object;
-import software.amazon.awssdk.services.s3.model.S3Response;
-
public class ExternalDataUtils {
private static final Map<ATypeTag, IValueParserFactory> valueParserFactoryMap = new EnumMap<>(ATypeTag.class);
private static final int DEFAULT_MAX_ARGUMENT_SZ = 1024 * 1024;
@@ -323,7 +242,7 @@
public static boolean isTrue(Map<String, String> configuration, String key) {
String value = configuration.get(key);
- return value == null ? false : Boolean.valueOf(value);
+ return value != null && Boolean.valueOf(value);
}
// Currently not used.
@@ -606,16 +525,16 @@
switch (type) {
case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
- AwsS3.validateProperties(configuration, srcLoc, collector);
+ S3Utils.validateProperties(configuration, srcLoc, collector);
break;
case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB:
- Azure.validateAzureBlobProperties(configuration, srcLoc, collector, appCtx);
+ validateAzureBlobProperties(configuration, srcLoc, collector, appCtx);
break;
case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_DATA_LAKE:
- Azure.validateAzureDataLakeProperties(configuration, srcLoc, collector, appCtx);
+ validateAzureDataLakeProperties(configuration, srcLoc, collector, appCtx);
break;
- case KEY_ADAPTER_NAME_GCS:
- GCS.validateProperties(configuration, srcLoc, collector);
+ case ExternalDataConstants.KEY_ADAPTER_NAME_GCS:
+ validateProperties(configuration, srcLoc, collector);
break;
default:
// Nothing needs to be done
@@ -864,15 +783,15 @@
}
}
- private static boolean isParquetFormat(Map<String, String> properties) {
+ public static boolean isParquetFormat(Map<String, String> properties) {
String inputFormat = properties.get(ExternalDataConstants.KEY_INPUT_FORMAT);
return ExternalDataConstants.CLASS_NAME_PARQUET_INPUT_FORMAT.equals(inputFormat)
|| ExternalDataConstants.INPUT_FORMAT_PARQUET.equals(inputFormat)
|| ExternalDataConstants.FORMAT_PARQUET.equals(properties.get(ExternalDataConstants.KEY_FORMAT));
}
- public static void setExternalDataProjectionInfo(DataProjectionInfo projectionInfo, Map<String, String> properties)
- throws IOException {
+ public static void setExternalDataProjectionInfo(DataProjectionFiltrationInfo projectionInfo,
+ Map<String, String> properties) throws IOException {
properties.put(ExternalDataConstants.KEY_REQUESTED_FIELDS,
serializeExpectedTypeToString(projectionInfo.getProjectionInfo()));
properties.put(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION,
@@ -886,14 +805,15 @@
* @return the expected type as Base64 string
*/
private static String serializeExpectedTypeToString(ARecordType expectedType) throws IOException {
- if (expectedType == DataProjectionInfo.EMPTY_TYPE || expectedType == DataProjectionInfo.ALL_FIELDS_TYPE) {
+ if (expectedType == DataProjectionFiltrationInfo.EMPTY_TYPE
+ || expectedType == DataProjectionFiltrationInfo.ALL_FIELDS_TYPE) {
//Return the type name of EMPTY_TYPE and ALL_FIELDS_TYPE
return expectedType.getTypeName();
}
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
Base64.Encoder encoder = Base64.getEncoder();
- DataProjectionInfo.writeTypeField(expectedType, dataOutputStream);
+ DataProjectionFiltrationInfo.writeTypeField(expectedType, dataOutputStream);
return encoder.encodeToString(byteArrayOutputStream.toByteArray());
}
@@ -909,1039 +829,10 @@
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
Base64.Encoder encoder = Base64.getEncoder();
- DataProjectionInfo.writeFunctionCallInformationMapField(functionCallInfoMap, dataOutputStream);
+ DataProjectionFiltrationInfo.writeFunctionCallInformationMapField(functionCallInfoMap, dataOutputStream);
return encoder.encodeToString(byteArrayOutputStream.toByteArray());
}
- public static class AwsS3 {
- private AwsS3() {
- throw new AssertionError("do not instantiate");
- }
-
- /**
- * Builds the S3 client using the provided configuration
- *
- * @param configuration properties
- * @return S3 client
- * @throws CompilationException CompilationException
- */
- public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
- // TODO(Hussain): Need to ensure that all required parameters are present in a previous step
- String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
- String sessionToken = configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME);
- String regionId = configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME);
- String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME);
-
- S3ClientBuilder builder = S3Client.builder();
-
- // Credentials
- AwsCredentialsProvider credentialsProvider;
-
- // No auth required
- if (accessKeyId == null) {
- credentialsProvider = AnonymousCredentialsProvider.create();
- } else {
- // auth required, check for temporary or permanent credentials
- if (sessionToken != null) {
- credentialsProvider = StaticCredentialsProvider
- .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
- } else {
- credentialsProvider =
- StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
- }
- }
-
- builder.credentialsProvider(credentialsProvider);
-
- // Validate the region
- List<Region> regions = S3Client.serviceMetadata().regions();
- Optional<Region> selectedRegion =
- regions.stream().filter(region -> region.id().equals(regionId)).findFirst();
-
- if (selectedRegion.isEmpty()) {
- throw new CompilationException(S3_REGION_NOT_SUPPORTED, regionId);
- }
- builder.region(selectedRegion.get());
-
- // Validate the service endpoint if present
- if (serviceEndpoint != null) {
- try {
- URI uri = new URI(serviceEndpoint);
- try {
- builder.endpointOverride(uri);
- } catch (NullPointerException ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- } catch (URISyntaxException ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR,
- String.format("Invalid service endpoint %s", serviceEndpoint));
- }
- }
-
- return builder.build();
- }
-
- /**
- * Builds the S3 client using the provided configuration
- *
- * @param configuration properties
- * @param numberOfPartitions number of partitions in the cluster
- */
- public static void configureAwsS3HdfsJobConf(JobConf conf, Map<String, String> configuration,
- int numberOfPartitions) {
- String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey = configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME);
- String sessionToken = configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME);
- String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME);
-
- //Disable caching S3 FileSystem
- HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_S3_PROTOCOL);
-
- /*
- * Authentication Methods:
- * 1- Anonymous: no accessKeyId and no secretAccessKey
- * 2- Temporary: has to provide accessKeyId, secretAccessKey and sessionToken
- * 3- Private: has to provide accessKeyId and secretAccessKey
- */
- if (accessKeyId == null) {
- //Tells hadoop-aws it is an anonymous access
- conf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_ANONYMOUS_ACCESS);
- } else {
- conf.set(HADOOP_ACCESS_KEY_ID, accessKeyId);
- conf.set(HADOOP_SECRET_ACCESS_KEY, secretAccessKey);
- if (sessionToken != null) {
- conf.set(HADOOP_SESSION_TOKEN, sessionToken);
- //Tells hadoop-aws it is a temporary access
- conf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_TEMP_ACCESS);
- }
- }
-
- /*
- * This is to allow S3 definition to have path-style form. Should always be true to match the current
- * way we access files in S3
- */
- conf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
-
- /*
- * Set the size of S3 connection pool to be the number of partitions
- */
- conf.set(HADOOP_S3_CONNECTION_POOL_SIZE, String.valueOf(numberOfPartitions));
-
- if (serviceEndpoint != null) {
- // Validation of the URL should be done at hadoop-aws level
- conf.set(ExternalDataConstants.AwsS3.HADOOP_SERVICE_END_POINT, serviceEndpoint);
- } else {
- //Region is ignored and buckets could be found by the central endpoint
- conf.set(ExternalDataConstants.AwsS3.HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT);
- }
- }
-
- /**
- * Validate external dataset properties
- *
- * @param configuration properties
- * @throws CompilationException Compilation exception
- */
- public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc,
- IWarningCollector collector) throws CompilationException {
-
- // check if the format property is present
- if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
- throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
- }
-
- // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
- String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
- if (accessKeyId == null || secretAccessKey == null) {
- // If one is passed, the other is required
- if (accessKeyId != null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
- ACCESS_KEY_ID_FIELD_NAME);
- } else if (secretAccessKey != null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
- SECRET_ACCESS_KEY_FIELD_NAME);
- }
- }
-
- validateIncludeExclude(configuration);
-
- // Check if the bucket is present
- S3Client s3Client = buildAwsS3Client(configuration);
- S3Response response;
- boolean useOldApi = false;
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- String prefix = getPrefix(configuration);
-
- try {
- response = isBucketEmpty(s3Client, container, prefix, false);
- } catch (S3Exception ex) {
- // Method not implemented, try falling back to old API
- try {
- // For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
- useOldApi = true;
- response = isBucketEmpty(s3Client, container, prefix, true);
- } else {
- throw ex;
- }
- } catch (SdkException ex2) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- } catch (SdkException ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- } finally {
- if (s3Client != null) {
- CleanupUtils.close(s3Client, null);
- }
- }
-
- boolean isEmpty = useOldApi ? ((ListObjectsResponse) response).contents().isEmpty()
- : ((ListObjectsV2Response) response).contents().isEmpty();
- if (isEmpty && collector.shouldWarn()) {
- Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- collector.warn(warning);
- }
-
- // Returns 200 only in case the bucket exists, otherwise, throws an exception. However, to
- // ensure coverage, check if the result is successful as well and not only catch exceptions
- if (!response.sdkHttpResponse().isSuccessful()) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_CONTAINER_NOT_FOUND, container);
- }
- }
-
- /**
- * Checks for a single object in the specified bucket to determine if the bucket is empty or not.
- *
- * @param s3Client s3 client
- * @param container the container name
- * @param prefix Prefix to be used
- * @param useOldApi flag whether to use the old API or not
- * @return returns the S3 response
- */
- private static S3Response isBucketEmpty(S3Client s3Client, String container, String prefix, boolean useOldApi) {
- S3Response response;
- if (useOldApi) {
- ListObjectsRequest.Builder listObjectsBuilder = ListObjectsRequest.builder();
- listObjectsBuilder.prefix(prefix);
- response = s3Client.listObjects(listObjectsBuilder.bucket(container).maxKeys(1).build());
- } else {
- ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder();
- listObjectsBuilder.prefix(prefix);
- response = s3Client.listObjectsV2(listObjectsBuilder.bucket(container).maxKeys(1).build());
- }
- return response;
- }
-
- /**
- * Returns the lists of S3 objects.
- *
- * @param configuration properties
- * @param includeExcludeMatcher include/exclude matchers to apply
- */
- public static List<S3Object> listS3Objects(Map<String, String> configuration,
- IncludeExcludeMatcher includeExcludeMatcher, IWarningCollector warningCollector)
- throws CompilationException {
- // Prepare to retrieve the objects
- List<S3Object> filesOnly;
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- S3Client s3Client = buildAwsS3Client(configuration);
- String prefix = getPrefix(configuration);
-
- try {
- filesOnly = listS3Objects(s3Client, container, prefix, includeExcludeMatcher);
- } catch (S3Exception ex) {
- // New API is not implemented, try falling back to old API
- try {
- // For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode()
- .equals(ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED)) {
- filesOnly = oldApiListS3Objects(s3Client, container, prefix, includeExcludeMatcher);
- } else {
- throw ex;
- }
- } catch (SdkException ex2) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- } catch (SdkException ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- } finally {
- if (s3Client != null) {
- CleanupUtils.close(s3Client, null);
- }
- }
-
- // Warn if no files are returned
- if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
- Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- warningCollector.warn(warning);
- }
-
- return filesOnly;
- }
-
- /**
- * Uses the latest API to retrieve the objects from the storage.
- *
- * @param s3Client S3 client
- * @param container container name
- * @param prefix definition prefix
- * @param includeExcludeMatcher include/exclude matchers to apply
- */
- private static List<S3Object> listS3Objects(S3Client s3Client, String container, String prefix,
- IncludeExcludeMatcher includeExcludeMatcher) {
- String newMarker = null;
- List<S3Object> filesOnly = new ArrayList<>();
-
- ListObjectsV2Response listObjectsResponse;
- ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(container);
- listObjectsBuilder.prefix(prefix);
-
- while (true) {
- // List the objects from the start, or from the last marker in case of truncated result
- if (newMarker == null) {
- listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.build());
- } else {
- listObjectsResponse =
- s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build());
- }
-
- // Collect the paths to files only
- collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
- includeExcludeMatcher.getMatchersList(), filesOnly);
-
- // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
- if (!listObjectsResponse.isTruncated()) {
- break;
- } else {
- newMarker = listObjectsResponse.nextContinuationToken();
- }
- }
-
- return filesOnly;
- }
-
- /**
- * Uses the old API (in case the new API is not implemented) to retrieve the objects from the storage
- *
- * @param s3Client S3 client
- * @param container container name
- * @param prefix definition prefix
- * @param includeExcludeMatcher include/exclude matchers to apply
- */
- private static List<S3Object> oldApiListS3Objects(S3Client s3Client, String container, String prefix,
- IncludeExcludeMatcher includeExcludeMatcher) {
- String newMarker = null;
- List<S3Object> filesOnly = new ArrayList<>();
-
- ListObjectsResponse listObjectsResponse;
- ListObjectsRequest.Builder listObjectsBuilder = ListObjectsRequest.builder().bucket(container);
- listObjectsBuilder.prefix(prefix);
-
- while (true) {
- // List the objects from the start, or from the last marker in case of truncated result
- if (newMarker == null) {
- listObjectsResponse = s3Client.listObjects(listObjectsBuilder.build());
- } else {
- listObjectsResponse = s3Client.listObjects(listObjectsBuilder.marker(newMarker).build());
- }
-
- // Collect the paths to files only
- collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
- includeExcludeMatcher.getMatchersList(), filesOnly);
-
- // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
- if (!listObjectsResponse.isTruncated()) {
- break;
- } else {
- newMarker = listObjectsResponse.nextMarker();
- }
- }
-
- return filesOnly;
- }
-
- /**
- * AWS S3 returns all the objects as paths, not differentiating between folder and files. The path is considered
- * a file if it does not end up with a "/" which is the separator in a folder structure.
- *
- * @param s3Objects List of returned objects
- */
- private static void collectAndFilterFiles(List<S3Object> s3Objects,
- BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers, List<S3Object> filesOnly) {
- for (S3Object object : s3Objects) {
- // skip folders
- if (object.key().endsWith("/")) {
- continue;
- }
-
- // No filter, add file
- if (predicate.test(matchers, object.key())) {
- filesOnly.add(object);
- }
- }
- }
- }
-
- /*
- * Note: Azure Blob and Azure Datalake use identical authentication, so they are using the same properties.
- * If they end up diverging, then properties for AzureBlob and AzureDataLake need to be created.
- */
- public static class Azure {
- private Azure() {
- throw new AssertionError("do not instantiate");
- }
-
- /**
- * Builds the Azure storage account using the provided configuration
- *
- * @param configuration properties
- * @return client
- */
- public static BlobServiceClient buildAzureBlobClient(IApplicationContext appCtx,
- Map<String, String> configuration) throws CompilationException {
- String managedIdentityId = configuration.get(MANAGED_IDENTITY_ID_FIELD_NAME);
- String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
- String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
- String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
- String tenantId = configuration.get(TENANT_ID_FIELD_NAME);
- String clientId = configuration.get(CLIENT_ID_FIELD_NAME);
- String clientSecret = configuration.get(CLIENT_SECRET_FIELD_NAME);
- String clientCertificate = configuration.get(CLIENT_CERTIFICATE_FIELD_NAME);
- String clientCertificatePassword = configuration.get(CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME);
- String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
-
- // Client builder
- BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
- int timeout = appCtx.getExternalProperties().getAzureRequestTimeout();
- RequestRetryOptions requestRetryOptions = new RequestRetryOptions(null, null, timeout, null, null, null);
- builder.retryOptions(requestRetryOptions);
-
- // Endpoint is required
- if (endpoint == null) {
- throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME);
- }
- builder.endpoint(endpoint);
-
- // Shared Key
- if (accountName != null || accountKey != null) {
- if (accountName == null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_NAME_FIELD_NAME,
- ACCOUNT_KEY_FIELD_NAME);
- }
-
- if (accountKey == null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_KEY_FIELD_NAME,
- ACCOUNT_NAME_FIELD_NAME);
- }
-
- Optional<String> provided = getFirstNotNull(configuration, SHARED_ACCESS_SIGNATURE_FIELD_NAME,
- MANAGED_IDENTITY_ID_FIELD_NAME, CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME,
- CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- ACCOUNT_KEY_FIELD_NAME);
- }
- StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey);
- builder.credential(credential);
- }
-
- // Shared access signature
- if (sharedAccessSignature != null) {
- Optional<String> provided = getFirstNotNull(configuration, MANAGED_IDENTITY_ID_FIELD_NAME,
- CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME,
- CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- SHARED_ACCESS_SIGNATURE_FIELD_NAME);
- }
- AzureSasCredential credential = new AzureSasCredential(sharedAccessSignature);
- builder.credential(credential);
- }
-
- // Managed Identity auth
- if (managedIdentityId != null) {
- Optional<String> provided = getFirstNotNull(configuration, CLIENT_ID_FIELD_NAME,
- CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME,
- TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- MANAGED_IDENTITY_ID_FIELD_NAME);
- }
- builder.credential(new ManagedIdentityCredentialBuilder().clientId(managedIdentityId).build());
- }
-
- // Client secret & certificate auth
- if (clientId != null) {
- // Both (or neither) client secret and client secret were provided, only one is allowed
- if ((clientSecret == null) == (clientCertificate == null)) {
- if (clientSecret != null) {
- throw new CompilationException(PARAMETERS_NOT_ALLOWED_AT_SAME_TIME, CLIENT_SECRET_FIELD_NAME,
- CLIENT_CERTIFICATE_FIELD_NAME);
- } else {
- throw new CompilationException(REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT,
- CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_ID_FIELD_NAME);
- }
- }
-
- // Tenant ID is required
- if (tenantId == null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, TENANT_ID_FIELD_NAME,
- CLIENT_ID_FIELD_NAME);
- }
-
- // Client certificate password is not allowed if client secret is used
- if (clientCertificatePassword != null && clientSecret != null) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
- CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, CLIENT_SECRET_FIELD_NAME);
- }
-
- // Use AD authentication
- if (clientSecret != null) {
- ClientSecretCredentialBuilder secret = new ClientSecretCredentialBuilder();
- secret.clientId(clientId);
- secret.tenantId(tenantId);
- secret.clientSecret(clientSecret);
- builder.credential(secret.build());
- } else {
- // Certificate
- ClientCertificateCredentialBuilder certificate = new ClientCertificateCredentialBuilder();
- certificate.clientId(clientId);
- certificate.tenantId(tenantId);
- try {
- InputStream certificateContent = new ByteArrayInputStream(clientCertificate.getBytes(UTF_8));
- if (clientCertificatePassword == null) {
- Method pemCertificate = ClientCertificateCredentialBuilder.class
- .getDeclaredMethod("pemCertificate", InputStream.class);
- pemCertificate.setAccessible(true);
- pemCertificate.invoke(certificate, certificateContent);
- } else {
- Method pemCertificate = ClientCertificateCredentialBuilder.class
- .getDeclaredMethod("pfxCertificate", InputStream.class, String.class);
- pemCertificate.setAccessible(true);
- pemCertificate.invoke(certificate, certificateContent, clientCertificatePassword);
- }
- } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ex) {
- throw new CompilationException(EXTERNAL_SOURCE_ERROR, ex.getMessage());
- }
- builder.credential(certificate.build());
- }
- }
-
- // If client id is not present, ensure client secret, certificate, tenant id and client certificate
- // password are not present
- if (clientId == null) {
- Optional<String> provided = getFirstNotNull(configuration, CLIENT_SECRET_FIELD_NAME,
- CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- SHARED_ACCESS_SIGNATURE_FIELD_NAME);
- }
- }
-
- try {
- return builder.buildClient();
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- }
-
- /**
- * Builds the Azure data lake storage account using the provided configuration
- *
- * @param configuration properties
- * @return client
- */
- public static DataLakeServiceClient buildAzureDatalakeClient(IApplicationContext appCtx,
- Map<String, String> configuration) throws CompilationException {
- String managedIdentityId = configuration.get(MANAGED_IDENTITY_ID_FIELD_NAME);
- String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
- String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
- String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
- String tenantId = configuration.get(TENANT_ID_FIELD_NAME);
- String clientId = configuration.get(CLIENT_ID_FIELD_NAME);
- String clientSecret = configuration.get(CLIENT_SECRET_FIELD_NAME);
- String clientCertificate = configuration.get(CLIENT_CERTIFICATE_FIELD_NAME);
- String clientCertificatePassword = configuration.get(CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME);
- String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
-
- // Client builder
- DataLakeServiceClientBuilder builder = new DataLakeServiceClientBuilder();
- int timeout = appCtx.getExternalProperties().getAzureRequestTimeout();
- RequestRetryOptions requestRetryOptions = new RequestRetryOptions(null, null, timeout, null, null, null);
- builder.retryOptions(requestRetryOptions);
-
- // Endpoint is required
- if (endpoint == null) {
- throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME);
- }
- builder.endpoint(endpoint);
-
- // Shared Key
- if (accountName != null || accountKey != null) {
- if (accountName == null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_NAME_FIELD_NAME,
- ACCOUNT_KEY_FIELD_NAME);
- }
-
- if (accountKey == null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_KEY_FIELD_NAME,
- ACCOUNT_NAME_FIELD_NAME);
- }
-
- Optional<String> provided = getFirstNotNull(configuration, SHARED_ACCESS_SIGNATURE_FIELD_NAME,
- MANAGED_IDENTITY_ID_FIELD_NAME, CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME,
- CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- ACCOUNT_KEY_FIELD_NAME);
- }
- StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey);
- builder.credential(credential);
- }
-
- // Shared access signature
- if (sharedAccessSignature != null) {
- Optional<String> provided = getFirstNotNull(configuration, MANAGED_IDENTITY_ID_FIELD_NAME,
- CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME,
- CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- SHARED_ACCESS_SIGNATURE_FIELD_NAME);
- }
- AzureSasCredential credential = new AzureSasCredential(sharedAccessSignature);
- builder.credential(credential);
- }
-
- // Managed Identity auth
- if (managedIdentityId != null) {
- Optional<String> provided = getFirstNotNull(configuration, CLIENT_ID_FIELD_NAME,
- CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME,
- TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- MANAGED_IDENTITY_ID_FIELD_NAME);
- }
- builder.credential(new ManagedIdentityCredentialBuilder().clientId(managedIdentityId).build());
- }
-
- // Client secret & certificate auth
- if (clientId != null) {
- // Both (or neither) client secret and client secret were provided, only one is allowed
- if ((clientSecret == null) == (clientCertificate == null)) {
- if (clientSecret != null) {
- throw new CompilationException(PARAMETERS_NOT_ALLOWED_AT_SAME_TIME, CLIENT_SECRET_FIELD_NAME,
- CLIENT_CERTIFICATE_FIELD_NAME);
- } else {
- throw new CompilationException(REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT,
- CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_ID_FIELD_NAME);
- }
- }
-
- // Tenant ID is required
- if (tenantId == null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, TENANT_ID_FIELD_NAME,
- CLIENT_ID_FIELD_NAME);
- }
-
- // Client certificate password is not allowed if client secret is used
- if (clientCertificatePassword != null && clientSecret != null) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
- CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, CLIENT_SECRET_FIELD_NAME);
- }
-
- // Use AD authentication
- if (clientSecret != null) {
- ClientSecretCredentialBuilder secret = new ClientSecretCredentialBuilder();
- secret.clientId(clientId);
- secret.tenantId(tenantId);
- secret.clientSecret(clientSecret);
- builder.credential(secret.build());
- } else {
- // Certificate
- ClientCertificateCredentialBuilder certificate = new ClientCertificateCredentialBuilder();
- certificate.clientId(clientId);
- certificate.tenantId(tenantId);
- try {
- InputStream certificateContent = new ByteArrayInputStream(clientCertificate.getBytes(UTF_8));
- if (clientCertificatePassword == null) {
- Method pemCertificate = ClientCertificateCredentialBuilder.class
- .getDeclaredMethod("pemCertificate", InputStream.class);
- pemCertificate.setAccessible(true);
- pemCertificate.invoke(certificate, certificateContent);
- } else {
- Method pemCertificate = ClientCertificateCredentialBuilder.class
- .getDeclaredMethod("pfxCertificate", InputStream.class, String.class);
- pemCertificate.setAccessible(true);
- pemCertificate.invoke(certificate, certificateContent, clientCertificatePassword);
- }
- } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ex) {
- throw new CompilationException(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- builder.credential(certificate.build());
- }
- }
-
- // If client id is not present, ensure client secret, certificate, tenant id and client certificate
- // password are not present
- if (clientId == null) {
- Optional<String> provided = getFirstNotNull(configuration, CLIENT_SECRET_FIELD_NAME,
- CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
- if (provided.isPresent()) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
- SHARED_ACCESS_SIGNATURE_FIELD_NAME);
- }
- }
-
- try {
- return builder.buildClient();
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- }
-
- public static List<BlobItem> listBlobItems(BlobServiceClient blobServiceClient,
- Map<String, String> configuration, IncludeExcludeMatcher includeExcludeMatcher,
- IWarningCollector warningCollector) throws CompilationException {
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
-
- List<BlobItem> filesOnly = new ArrayList<>();
-
- // Ensure the validity of include/exclude
- ExternalDataUtils.validateIncludeExclude(configuration);
-
- BlobContainerClient blobContainer;
- try {
- blobContainer = blobServiceClient.getBlobContainerClient(container);
-
- // Get all objects in a container and extract the paths to files
- ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
- listBlobsOptions.setPrefix(ExternalDataUtils.getPrefix(configuration));
- Iterable<BlobItem> blobItems = blobContainer.listBlobs(listBlobsOptions, null);
-
- // Collect the paths to files only
- collectAndFilterBlobFiles(blobItems, includeExcludeMatcher.getPredicate(),
- includeExcludeMatcher.getMatchersList(), filesOnly);
-
- // Warn if no files are returned
- if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
- Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- warningCollector.warn(warning);
- }
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
-
- return filesOnly;
- }
-
- /**
- * Collects and filters the files only, and excludes any folders
- *
- * @param items storage items
- * @param predicate predicate to test with for file filtration
- * @param matchers include/exclude matchers to test against
- * @param filesOnly List containing the files only (excluding folders)
- */
- private static void collectAndFilterBlobFiles(Iterable<BlobItem> items,
- BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers, List<BlobItem> filesOnly) {
- for (BlobItem item : items) {
- String uri = item.getName();
-
- // skip folders
- if (uri.endsWith("/")) {
- continue;
- }
-
- // No filter, add file
- if (predicate.test(matchers, uri)) {
- filesOnly.add(item);
- }
- }
- }
-
- public static List<PathItem> listDatalakePathItems(DataLakeServiceClient client,
- Map<String, String> configuration, IncludeExcludeMatcher includeExcludeMatcher,
- IWarningCollector warningCollector) throws CompilationException {
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
-
- List<PathItem> filesOnly = new ArrayList<>();
-
- // Ensure the validity of include/exclude
- ExternalDataUtils.validateIncludeExclude(configuration);
-
- DataLakeFileSystemClient fileSystemClient;
- try {
- fileSystemClient = client.getFileSystemClient(container);
-
- // Get all objects in a container and extract the paths to files
- ListPathsOptions listOptions = new ListPathsOptions();
- boolean recursive = Boolean.parseBoolean(configuration.get(RECURSIVE_FIELD_NAME));
- listOptions.setRecursive(recursive);
- listOptions.setPath(ExternalDataUtils.getPrefix(configuration, false));
- PagedIterable<PathItem> pathItems = fileSystemClient.listPaths(listOptions, null);
-
- // Collect the paths to files only
- collectAndFilterDatalakeFiles(pathItems, includeExcludeMatcher.getPredicate(),
- includeExcludeMatcher.getMatchersList(), filesOnly);
-
- // Warn if no files are returned
- if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
- Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- warningCollector.warn(warning);
- }
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
-
- return filesOnly;
- }
-
- /**
- * Collects and filters the files only, and excludes any folders
- *
- * @param items storage items
- * @param predicate predicate to test with for file filtration
- * @param matchers include/exclude matchers to test against
- * @param filesOnly List containing the files only (excluding folders)
- */
- private static void collectAndFilterDatalakeFiles(Iterable<PathItem> items,
- BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers, List<PathItem> filesOnly) {
- for (PathItem item : items) {
- String uri = item.getName();
-
- // skip folders
- if (uri.endsWith("/")) {
- continue;
- }
-
- // No filter, add file
- if (predicate.test(matchers, uri)) {
- filesOnly.add(item);
- }
- }
- }
-
- /**
- * Validate external dataset properties
- *
- * @param configuration properties
- * @throws CompilationException Compilation exception
- */
- public static void validateAzureBlobProperties(Map<String, String> configuration, SourceLocation srcLoc,
- IWarningCollector collector, IApplicationContext appCtx) throws CompilationException {
-
- // check if the format property is present
- if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
- throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
- }
-
- validateIncludeExclude(configuration);
-
- // Check if the bucket is present
- BlobServiceClient blobServiceClient;
- try {
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- blobServiceClient = buildAzureBlobClient(appCtx, configuration);
- BlobContainerClient blobContainer = blobServiceClient.getBlobContainerClient(container);
-
- // Get all objects in a container and extract the paths to files
- ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
- listBlobsOptions.setPrefix(getPrefix(configuration));
- Iterable<BlobItem> blobItems = blobContainer.listBlobs(listBlobsOptions, null);
-
- if (!blobItems.iterator().hasNext() && collector.shouldWarn()) {
- Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- collector.warn(warning);
- }
- } catch (CompilationException ex) {
- throw ex;
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- }
-
- /**
- * Validate external dataset properties
- *
- * @param configuration properties
- * @throws CompilationException Compilation exception
- */
- public static void validateAzureDataLakeProperties(Map<String, String> configuration, SourceLocation srcLoc,
- IWarningCollector collector, IApplicationContext appCtx) throws CompilationException {
-
- // check if the format property is present
- if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
- throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
- }
-
- validateIncludeExclude(configuration);
-
- // Check if the bucket is present
- DataLakeServiceClient dataLakeServiceClient;
- try {
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- dataLakeServiceClient = buildAzureDatalakeClient(appCtx, configuration);
- DataLakeFileSystemClient fileSystemClient = dataLakeServiceClient.getFileSystemClient(container);
-
- // Get all objects in a container and extract the paths to files
- ListPathsOptions listPathsOptions = new ListPathsOptions();
- listPathsOptions.setPath(getPrefix(configuration));
- Iterable<PathItem> blobItems = fileSystemClient.listPaths(listPathsOptions, null);
-
- if (!blobItems.iterator().hasNext() && collector.shouldWarn()) {
- Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- collector.warn(warning);
- }
- } catch (CompilationException ex) {
- throw ex;
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- }
-
- /**
- * Builds the Azure Blob storage client using the provided configuration
- *
- * @param configuration properties
- * @see <a href="https://docs.microsoft.com/en-us/azure/databricks/data/data-sources/azure/azure-storage">Azure
- * Blob storage</a>
- */
- public static void configureAzureHdfsJobConf(JobConf conf, Map<String, String> configuration, String endPoint) {
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
- String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
-
- //Disable caching S3 FileSystem
- HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_AZURE_BLOB_PROTOCOL);
-
- //Key for Hadoop configuration
- StringBuilder hadoopKey = new StringBuilder();
- //Value for Hadoop configuration
- String hadoopValue;
- if (accountKey != null || sharedAccessSignature != null) {
- if (accountKey != null) {
- hadoopKey.append(HADOOP_AZURE_FS_ACCOUNT_KEY).append('.');
- //Set only the AccountKey
- hadoopValue = accountKey;
- } else {
- //Use SAS for Hadoop FS as connectionString is provided
- hadoopKey.append(HADOOP_AZURE_FS_SAS).append('.');
- //Setting the container is required for SAS
- hadoopKey.append(container).append('.');
- //Set the connection string for SAS
- hadoopValue = sharedAccessSignature;
- }
- //Set the endPoint, which includes the AccountName
- hadoopKey.append(endPoint);
- //Tells Hadoop we are reading from Blob Storage
- conf.set(hadoopKey.toString(), hadoopValue);
- }
- }
- }
-
- public static class GCS {
- private GCS() {
- throw new AssertionError("do not instantiate");
-
- }
-
- //TODO(htowaileb): Add validation step similar to other externals, which also checks if empty bucket
- //upon creating the external dataset
-
- /**
- * Builds the client using the provided configuration
- *
- * @param configuration properties
- * @return clientasterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
- * @throws CompilationException CompilationException
- */
- public static Storage buildClient(Map<String, String> configuration) throws CompilationException {
- String applicationDefaultCredentials = configuration.get(APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
- String jsonCredentials = configuration.get(JSON_CREDENTIALS_FIELD_NAME);
- String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
-
- StorageOptions.Builder builder = StorageOptions.newBuilder();
-
- // default credentials provider
- if (applicationDefaultCredentials != null) {
- // only "true" value is allowed
- if (!applicationDefaultCredentials.equalsIgnoreCase("true")) {
- throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE,
- APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME, "true");
- }
-
- // no other authentication parameters are allowed
- if (jsonCredentials != null) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, JSON_CREDENTIALS_FIELD_NAME,
- APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
- }
-
- try {
- builder.setCredentials(GoogleCredentials.getApplicationDefault());
- } catch (IOException ex) {
- throw CompilationException.create(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- }
-
- // json credentials
- if (jsonCredentials != null) {
- try (InputStream credentialsStream = new ByteArrayInputStream(jsonCredentials.getBytes())) {
- builder.setCredentials(GoogleCredentials.fromStream(credentialsStream));
- } catch (IOException ex) {
- throw new CompilationException(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- }
-
- if (endpoint != null) {
- builder.setHost(endpoint);
- }
-
- return builder.build().getService();
- }
-
- /**
- * Validate external dataset properties
- *
- * @param configuration properties
- * @throws CompilationException Compilation exception
- */
- public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc,
- IWarningCollector collector) throws CompilationException {
-
- // check if the format property is present
- if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
- throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
- }
-
- // parquet is not supported for google cloud storage
- if (isParquetFormat(configuration)) {
- throw new CompilationException(INVALID_REQ_PARAM_VAL, srcLoc, KEY_FORMAT,
- configuration.get(KEY_FORMAT));
- }
-
- validateIncludeExclude(configuration);
- String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
-
- try {
- BlobListOption limitOption = BlobListOption.pageSize(1);
- BlobListOption prefixOption = BlobListOption.prefix(getPrefix(configuration));
- Storage storage = buildClient(configuration);
- Page<Blob> items = storage.list(container, limitOption, prefixOption);
-
- if (!items.iterateAll().iterator().hasNext() && collector.shouldWarn()) {
- Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
- collector.warn(warning);
- }
- } catch (CompilationException ex) {
- throw ex;
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
- }
- }
- }
-
public static int roundUpToNearestFrameSize(int size, int framesize) {
return ((size / framesize) + 1) * framesize;
}
@@ -1958,7 +849,25 @@
return maxArgSz;
}
- private static Optional<String> getFirstNotNull(Map<String, String> configuration, String... parameters) {
+ public static Optional<String> getFirstNotNull(Map<String, String> configuration, String... parameters) {
return Arrays.stream(parameters).filter(field -> configuration.get(field) != null).findFirst();
}
+
+ public static ATypeTag peekArgument(IAType type, IValueReference valueReference, TaggedValuePointable pointy)
+ throws HyracksDataException {
+ ATypeTag tag = type.getTypeTag();
+ if (tag == ATypeTag.ANY) {
+ pointy.set(valueReference);
+ ATypeTag rtTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(pointy.getTag());
+ IAType rtType = TypeTagUtil.getBuiltinTypeByTag(rtTypeTag);
+ return MessagePackUtils.peekUnknown(rtType);
+ } else {
+ return MessagePackUtils.peekUnknown(type);
+ }
+ }
+
+ public static void setVoidArgument(ArrayBackedValueStorage argHolder) throws IOException {
+ argHolder.getDataOutput().writeByte(ARRAY16);
+ argHolder.getDataOutput().writeShort((short) 0);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
index 3506216..a4e47ca 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
@@ -42,7 +42,7 @@
import org.apache.asterix.external.input.stream.HDFSInputStream;
import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
import org.apache.asterix.runtime.projection.FunctionCallInformation;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
@@ -242,7 +242,7 @@
String requestedValues = configuration.get(ExternalDataConstants.KEY_REQUESTED_FIELDS);
if (requestedValues == null) {
//No value is requested, return the entire record
- requestedValues = DataProjectionInfo.ALL_FIELDS_TYPE.getTypeName();
+ requestedValues = DataProjectionFiltrationInfo.ALL_FIELDS_TYPE.getTypeName();
} else {
//Subset of the values were requested, set the functionCallInformation
conf.set(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION,
@@ -284,18 +284,18 @@
public static ARecordType getExpectedType(Configuration configuration) throws IOException {
String encoded = configuration.get(ExternalDataConstants.KEY_REQUESTED_FIELDS, "");
- if (encoded.isEmpty() || encoded.equals(DataProjectionInfo.ALL_FIELDS_TYPE.getTypeName())) {
+ if (encoded.isEmpty() || encoded.equals(DataProjectionFiltrationInfo.ALL_FIELDS_TYPE.getTypeName())) {
//By default, return the entire records
- return DataProjectionInfo.ALL_FIELDS_TYPE;
- } else if (encoded.equals(DataProjectionInfo.EMPTY_TYPE.getTypeName())) {
+ return DataProjectionFiltrationInfo.ALL_FIELDS_TYPE;
+ } else if (encoded.equals(DataProjectionFiltrationInfo.EMPTY_TYPE.getTypeName())) {
//No fields were requested
- return DataProjectionInfo.EMPTY_TYPE;
+ return DataProjectionFiltrationInfo.EMPTY_TYPE;
}
//A subset of the fields was requested
Base64.Decoder decoder = Base64.getDecoder();
byte[] typeBytes = decoder.decode(encoded);
DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(typeBytes));
- return DataProjectionInfo.createTypeField(dataInputStream);
+ return DataProjectionFiltrationInfo.createTypeField(dataInputStream);
}
public static void setFunctionCallInformationMap(Map<String, FunctionCallInformation> funcCallInfoMap,
@@ -311,7 +311,7 @@
Base64.Decoder decoder = Base64.getDecoder();
byte[] functionCallInfoMapBytes = decoder.decode(encoded);
DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(functionCallInfoMapBytes));
- return DataProjectionInfo.createFunctionCallInformationMap(dataInputStream);
+ return DataProjectionFiltrationInfo.createFunctionCallInformationMap(dataInputStream);
}
return null;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
new file mode 100644
index 0000000..79bbbe2
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.aws.s3;
+
+public class S3Constants {
+ private S3Constants() {
+ throw new AssertionError("do not instantiate");
+ }
+
+ public static final String REGION_FIELD_NAME = "region";
+ public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
+ public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
+ public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
+ public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
+ public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
+
+ // AWS S3 specific error codes
+ public static final String ERROR_INTERNAL_ERROR = "InternalError";
+ public static final String ERROR_SLOW_DOWN = "SlowDown";
+ public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
+
+ /*
+ * Hadoop-AWS
+ * AWS connectors for s3 and s3n are deprecated.
+ */
+ public static final String HADOOP_ACCESS_KEY_ID = "fs.s3a.access.key";
+ public static final String HADOOP_SECRET_ACCESS_KEY = "fs.s3a.secret.key";
+ public static final String HADOOP_SESSION_TOKEN = "fs.s3a.session.token";
+ public static final String HADOOP_REGION = "fs.s3a.region";
+ public static final String HADOOP_SERVICE_END_POINT = "fs.s3a.endpoint";
+
+ /*
+ * Internal configurations
+ */
+ //Allows accessing directories as file system path
+ public static final String HADOOP_PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
+ //The number of maximum HTTP connections in connection pool
+ public static final String HADOOP_S3_CONNECTION_POOL_SIZE = "fs.s3a.connection.maximum";
+ //S3 used protocol
+ public static final String HADOOP_S3_PROTOCOL = "s3a";
+
+ //Hadoop credentials provider key
+ public static final String HADOOP_CREDENTIAL_PROVIDER_KEY = "fs.s3a.aws.credentials.provider";
+ //Anonymous credential provider
+ public static final String HADOOP_ANONYMOUS_ACCESS = "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider";
+ //Temporary credential provider
+ public static final String HADOOP_TEMP_ACCESS = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
new file mode 100644
index 0000000..6775bf12b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -0,0 +1,475 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.aws.s3;
+
+import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED;
+import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.*;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiPredicate;
+import java.util.regex.Matcher;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.HDFSUtils;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.util.CleanupUtils;
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.S3Response;
+
+public class S3Utils {
+ private S3Utils() {
+ throw new AssertionError("do not instantiate");
+ }
+
+ public static boolean isRetryableError(String errorCode) {
+ return errorCode.equals(ERROR_INTERNAL_ERROR) || errorCode.equals(ERROR_SLOW_DOWN);
+ }
+
+ /**
+ * Builds the S3 client using the provided configuration
+ *
+ * @param configuration properties
+ * @return S3 client
+ * @throws CompilationException CompilationException
+ */
+ public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
+ // TODO(Hussain): Need to ensure that all required parameters are present in a previous step
+ String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
+ String regionId = configuration.get(REGION_FIELD_NAME);
+ String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
+
+ S3ClientBuilder builder = S3Client.builder();
+
+ // Credentials
+ AwsCredentialsProvider credentialsProvider;
+
+ // nothing provided, anonymous authentication
+ if (instanceProfile == null && accessKeyId == null && secretAccessKey == null && sessionToken == null) {
+ credentialsProvider = AnonymousCredentialsProvider.create();
+ } else if (instanceProfile != null) {
+
+ // only "true" value is allowed
+ if (!instanceProfile.equalsIgnoreCase("true")) {
+ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, INSTANCE_PROFILE_FIELD_NAME, "true");
+ }
+
+ // no other authentication parameters are allowed
+ if (accessKeyId != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ INSTANCE_PROFILE_FIELD_NAME);
+ }
+ if (secretAccessKey != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+ INSTANCE_PROFILE_FIELD_NAME);
+ }
+ if (sessionToken != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, SESSION_TOKEN_FIELD_NAME,
+ INSTANCE_PROFILE_FIELD_NAME);
+ }
+ credentialsProvider = InstanceProfileCredentialsProvider.create();
+ } else if (accessKeyId != null || secretAccessKey != null) {
+ // accessKeyId authentication
+ if (accessKeyId == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SECRET_ACCESS_KEY_FIELD_NAME);
+ }
+ if (secretAccessKey == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+ ACCESS_KEY_ID_FIELD_NAME);
+ }
+
+ // use session token if provided
+ if (sessionToken != null) {
+ credentialsProvider = StaticCredentialsProvider
+ .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+ } else {
+ credentialsProvider =
+ StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+ }
+ } else {
+ // if only session token is provided, accessKeyId is required
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SESSION_TOKEN_FIELD_NAME);
+ }
+
+ builder.credentialsProvider(credentialsProvider);
+
+ // Validate the region
+ List<Region> regions = S3Client.serviceMetadata().regions();
+ Optional<Region> selectedRegion = regions.stream().filter(region -> region.id().equals(regionId)).findFirst();
+
+ if (selectedRegion.isEmpty()) {
+ throw new CompilationException(S3_REGION_NOT_SUPPORTED, regionId);
+ }
+ builder.region(selectedRegion.get());
+
+ // Validate the service endpoint if present
+ if (serviceEndpoint != null) {
+ try {
+ URI uri = new URI(serviceEndpoint);
+ try {
+ builder.endpointOverride(uri);
+ } catch (NullPointerException ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ } catch (URISyntaxException ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR,
+ String.format("Invalid service endpoint %s", serviceEndpoint));
+ }
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Builds the S3 client using the provided configuration
+ *
+ * @param configuration properties
+ * @param numberOfPartitions number of partitions in the cluster
+ */
+ public static void configureAwsS3HdfsJobConf(JobConf conf, Map<String, String> configuration,
+ int numberOfPartitions) {
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
+ String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
+
+ //Disable caching S3 FileSystem
+ HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_S3_PROTOCOL);
+
+ /*
+ * Authentication Methods:
+ * 1- Anonymous: no accessKeyId and no secretAccessKey
+ * 2- Temporary: has to provide accessKeyId, secretAccessKey and sessionToken
+ * 3- Private: has to provide accessKeyId and secretAccessKey
+ */
+ if (accessKeyId == null) {
+ //Tells hadoop-aws it is an anonymous access
+ conf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_ANONYMOUS_ACCESS);
+ } else {
+ conf.set(HADOOP_ACCESS_KEY_ID, accessKeyId);
+ conf.set(HADOOP_SECRET_ACCESS_KEY, secretAccessKey);
+ if (sessionToken != null) {
+ conf.set(HADOOP_SESSION_TOKEN, sessionToken);
+ //Tells hadoop-aws it is a temporary access
+ conf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_TEMP_ACCESS);
+ }
+ }
+
+ /*
+ * This is to allow S3 definition to have path-style form. Should always be true to match the current
+ * way we access files in S3
+ */
+ conf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
+
+ /*
+ * Set the size of S3 connection pool to be the number of partitions
+ */
+ conf.set(HADOOP_S3_CONNECTION_POOL_SIZE, String.valueOf(numberOfPartitions));
+
+ if (serviceEndpoint != null) {
+ // Validation of the URL should be done at hadoop-aws level
+ conf.set(HADOOP_SERVICE_END_POINT, serviceEndpoint);
+ } else {
+ //Region is ignored and buckets could be found by the central endpoint
+ conf.set(HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT);
+ }
+ }
+
+ /**
+ * Validate external dataset properties
+ *
+ * @param configuration properties
+ * @throws CompilationException Compilation exception
+ */
+ public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc,
+ IWarningCollector collector) throws CompilationException {
+
+ // check if the format property is present
+ if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+ throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
+ }
+
+ // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ if (accessKeyId == null || secretAccessKey == null) {
+ // If one is passed, the other is required
+ if (accessKeyId != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+ ACCESS_KEY_ID_FIELD_NAME);
+ } else if (secretAccessKey != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SECRET_ACCESS_KEY_FIELD_NAME);
+ }
+ }
+
+ validateIncludeExclude(configuration);
+
+ // Check if the bucket is present
+ S3Client s3Client = buildAwsS3Client(configuration);
+ S3Response response;
+ boolean useOldApi = false;
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+ String prefix = getPrefix(configuration);
+
+ try {
+ response = isBucketEmpty(s3Client, container, prefix, false);
+ } catch (S3Exception ex) {
+ // Method not implemented, try falling back to old API
+ try {
+ // For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+ if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
+ useOldApi = true;
+ response = isBucketEmpty(s3Client, container, prefix, true);
+ } else {
+ throw ex;
+ }
+ } catch (SdkException ex2) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ } catch (SdkException ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ } finally {
+ if (s3Client != null) {
+ CleanupUtils.close(s3Client, null);
+ }
+ }
+
+ boolean isEmpty = useOldApi ? ((ListObjectsResponse) response).contents().isEmpty()
+ : ((ListObjectsV2Response) response).contents().isEmpty();
+ if (isEmpty && collector.shouldWarn()) {
+ Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ collector.warn(warning);
+ }
+
+ // Returns 200 only in case the bucket exists, otherwise, throws an exception. However, to
+ // ensure coverage, check if the result is successful as well and not only catch exceptions
+ if (!response.sdkHttpResponse().isSuccessful()) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_CONTAINER_NOT_FOUND, container);
+ }
+ }
+
+ /**
+ * Checks for a single object in the specified bucket to determine if the bucket is empty or not.
+ *
+ * @param s3Client s3 client
+ * @param container the container name
+ * @param prefix Prefix to be used
+ * @param useOldApi flag whether to use the old API or not
+ * @return returns the S3 response
+ */
+ private static S3Response isBucketEmpty(S3Client s3Client, String container, String prefix, boolean useOldApi) {
+ S3Response response;
+ if (useOldApi) {
+ ListObjectsRequest.Builder listObjectsBuilder = ListObjectsRequest.builder();
+ listObjectsBuilder.prefix(prefix);
+ response = s3Client.listObjects(listObjectsBuilder.bucket(container).maxKeys(1).build());
+ } else {
+ ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder();
+ listObjectsBuilder.prefix(prefix);
+ response = s3Client.listObjectsV2(listObjectsBuilder.bucket(container).maxKeys(1).build());
+ }
+ return response;
+ }
+
+ /**
+ * Returns the lists of S3 objects.
+ *
+ * @param configuration properties
+ * @param includeExcludeMatcher include/exclude matchers to apply
+ */
+ public static List<S3Object> listS3Objects(Map<String, String> configuration,
+ AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
+ IWarningCollector warningCollector) throws CompilationException {
+ // Prepare to retrieve the objects
+ List<S3Object> filesOnly;
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+ S3Client s3Client = buildAwsS3Client(configuration);
+ String prefix = getPrefix(configuration);
+
+ try {
+ filesOnly = listS3Objects(s3Client, container, prefix, includeExcludeMatcher);
+ } catch (S3Exception ex) {
+ // New API is not implemented, try falling back to old API
+ try {
+ // For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+ if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
+ filesOnly = oldApiListS3Objects(s3Client, container, prefix, includeExcludeMatcher);
+ } else {
+ throw ex;
+ }
+ } catch (SdkException ex2) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ } catch (SdkException ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ } finally {
+ if (s3Client != null) {
+ CleanupUtils.close(s3Client, null);
+ }
+ }
+
+ // Warn if no files are returned
+ if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
+ Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ warningCollector.warn(warning);
+ }
+
+ return filesOnly;
+ }
+
+ /**
+ * Uses the latest API to retrieve the objects from the storage.
+ *
+ * @param s3Client S3 client
+ * @param container container name
+ * @param prefix definition prefix
+ * @param includeExcludeMatcher include/exclude matchers to apply
+ */
+ private static List<S3Object> listS3Objects(S3Client s3Client, String container, String prefix,
+ AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher) {
+ String newMarker = null;
+ List<S3Object> filesOnly = new ArrayList<>();
+
+ ListObjectsV2Response listObjectsResponse;
+ ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(container);
+ listObjectsBuilder.prefix(prefix);
+
+ while (true) {
+ // List the objects from the start, or from the last marker in case of truncated result
+ if (newMarker == null) {
+ listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.build());
+ } else {
+ listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build());
+ }
+
+ // Collect the paths to files only
+ collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
+ includeExcludeMatcher.getMatchersList(), filesOnly);
+
+ // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
+ if (!listObjectsResponse.isTruncated()) {
+ break;
+ } else {
+ newMarker = listObjectsResponse.nextContinuationToken();
+ }
+ }
+
+ return filesOnly;
+ }
+
+ /**
+ * Uses the old API (in case the new API is not implemented) to retrieve the objects from the storage
+ *
+ * @param s3Client S3 client
+ * @param container container name
+ * @param prefix definition prefix
+ * @param includeExcludeMatcher include/exclude matchers to apply
+ */
+ private static List<S3Object> oldApiListS3Objects(S3Client s3Client, String container, String prefix,
+ AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher) {
+ String newMarker = null;
+ List<S3Object> filesOnly = new ArrayList<>();
+
+ ListObjectsResponse listObjectsResponse;
+ ListObjectsRequest.Builder listObjectsBuilder = ListObjectsRequest.builder().bucket(container);
+ listObjectsBuilder.prefix(prefix);
+
+ while (true) {
+ // List the objects from the start, or from the last marker in case of truncated result
+ if (newMarker == null) {
+ listObjectsResponse = s3Client.listObjects(listObjectsBuilder.build());
+ } else {
+ listObjectsResponse = s3Client.listObjects(listObjectsBuilder.marker(newMarker).build());
+ }
+
+ // Collect the paths to files only
+ collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
+ includeExcludeMatcher.getMatchersList(), filesOnly);
+
+ // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
+ if (!listObjectsResponse.isTruncated()) {
+ break;
+ } else {
+ newMarker = listObjectsResponse.nextMarker();
+ }
+ }
+
+ return filesOnly;
+ }
+
+ /**
+ * AWS S3 returns all the objects as paths, not differentiating between folder and files. The path is considered
+ * a file if it does not end up with a "/" which is the separator in a folder structure.
+ *
+ * @param s3Objects List of returned objects
+ */
+ private static void collectAndFilterFiles(List<S3Object> s3Objects, BiPredicate<List<Matcher>, String> predicate,
+ List<Matcher> matchers, List<S3Object> filesOnly) {
+ for (S3Object object : s3Objects) {
+ // skip folders
+ if (object.key().endsWith("/")) {
+ continue;
+ }
+
+ // No filter, add file
+ if (predicate.test(matchers, object.key())) {
+ filesOnly.add(object);
+ }
+ }
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
new file mode 100644
index 0000000..9ade27b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.azure.blob_storage;
+
+/*
+ * Note: Azure Blob and Azure Datalake use identical authentication, so they are using the same properties.
+ * If they end up diverging, then properties for AzureBlob and AzureDataLake need to be created.
+ */
+public class AzureConstants {
+ private AzureConstants() {
+ throw new AssertionError("do not instantiate");
+ }
+
+ /*
+ * Asterix Configuration Keys
+ */
+ public static final String MANAGED_IDENTITY_ID_FIELD_NAME = "managedIdentityId";
+ public static final String ACCOUNT_NAME_FIELD_NAME = "accountName";
+ public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey";
+ public static final String SHARED_ACCESS_SIGNATURE_FIELD_NAME = "sharedAccessSignature";
+ public static final String TENANT_ID_FIELD_NAME = "tenantId";
+ public static final String CLIENT_ID_FIELD_NAME = "clientId";
+ public static final String CLIENT_SECRET_FIELD_NAME = "clientSecret";
+ public static final String CLIENT_CERTIFICATE_FIELD_NAME = "clientCertificate";
+ public static final String CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME = "clientCertificatePassword";
+ public static final String ENDPOINT_FIELD_NAME = "endpoint";
+
+ // Specific Azure data lake property
+ /*
+ The behavior of Data Lake (true file system) is to read the files of the specified prefix only, example:
+ storage/myData/personal/file1.json
+ storage/myData/personal/file2.json
+ storage/myData/file3.json
+ If the prefix used is "myData", then only the file file3.json is read. However, if the property "recursive"
+ is set to "true" when creating the external dataset, then it goes recursively overall the paths, and the result
+ is file1.json, file2.json and file3.json.
+ */
+ public static final String RECURSIVE_FIELD_NAME = "recursive";
+
+ /*
+ * Hadoop-Azure
+ */
+ //Used when accountName and accessKey are provided
+ public static final String HADOOP_AZURE_FS_ACCOUNT_KEY = "fs.azure.account.key";
+ //Used when a connectionString is provided
+ public static final String HADOOP_AZURE_FS_SAS = "fs.azure.sas";
+ public static final String HADOOP_AZURE_BLOB_PROTOCOL = "wasbs";
+ public static final String HADOOP_AZURE_DATALAKE_PROTOCOL = "abfss";
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
new file mode 100644
index 0000000..0dc9ad2
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
@@ -0,0 +1,636 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.azure.blob_storage;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_REQUIRED;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.external.util.ExternalDataUtils.getFirstNotNull;
+import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.ACCOUNT_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.ACCOUNT_NAME_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.CLIENT_CERTIFICATE_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.CLIENT_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.CLIENT_SECRET_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.ENDPOINT_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.HADOOP_AZURE_BLOB_PROTOCOL;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.HADOOP_AZURE_FS_ACCOUNT_KEY;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.HADOOP_AZURE_FS_SAS;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.MANAGED_IDENTITY_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.RECURSIVE_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
+import static org.apache.asterix.external.util.azure.blob_storage.AzureConstants.TENANT_ID_FIELD_NAME;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiPredicate;
+import java.util.regex.Matcher;
+
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.HDFSUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.exceptions.Warning;
+
+import com.azure.core.credential.AzureSasCredential;
+import com.azure.core.http.rest.PagedIterable;
+import com.azure.identity.ClientCertificateCredentialBuilder;
+import com.azure.identity.ClientSecretCredentialBuilder;
+import com.azure.identity.ManagedIdentityCredentialBuilder;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.ListBlobsOptions;
+import com.azure.storage.common.StorageSharedKeyCredential;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.azure.storage.file.datalake.DataLakeFileSystemClient;
+import com.azure.storage.file.datalake.DataLakeServiceClient;
+import com.azure.storage.file.datalake.DataLakeServiceClientBuilder;
+import com.azure.storage.file.datalake.models.ListPathsOptions;
+import com.azure.storage.file.datalake.models.PathItem;
+
+public class AzureUtils {
+ private AzureUtils() {
+ throw new AssertionError("do not instantiate");
+ }
+
+ /**
+ * Builds the Azure storage account using the provided configuration
+ *
+ * @param configuration properties
+ * @return client
+ */
+ public static BlobServiceClient buildAzureBlobClient(IApplicationContext appCtx, Map<String, String> configuration)
+ throws CompilationException {
+ String managedIdentityId = configuration.get(MANAGED_IDENTITY_ID_FIELD_NAME);
+ String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
+ String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
+ String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ String tenantId = configuration.get(TENANT_ID_FIELD_NAME);
+ String clientId = configuration.get(CLIENT_ID_FIELD_NAME);
+ String clientSecret = configuration.get(CLIENT_SECRET_FIELD_NAME);
+ String clientCertificate = configuration.get(CLIENT_CERTIFICATE_FIELD_NAME);
+ String clientCertificatePassword = configuration.get(CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME);
+ String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
+
+ // Client builder
+ BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
+ int timeout = appCtx.getExternalProperties().getAzureRequestTimeout();
+ RequestRetryOptions requestRetryOptions = new RequestRetryOptions(null, null, timeout, null, null, null);
+ builder.retryOptions(requestRetryOptions);
+
+ // Endpoint is required
+ if (endpoint == null) {
+ throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME);
+ }
+ builder.endpoint(endpoint);
+
+ // Shared Key
+ if (accountName != null || accountKey != null) {
+ if (accountName == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_NAME_FIELD_NAME,
+ ACCOUNT_KEY_FIELD_NAME);
+ }
+
+ if (accountKey == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_KEY_FIELD_NAME,
+ ACCOUNT_NAME_FIELD_NAME);
+ }
+
+ Optional<String> provided = getFirstNotNull(configuration, SHARED_ACCESS_SIGNATURE_FIELD_NAME,
+ MANAGED_IDENTITY_ID_FIELD_NAME, CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ ACCOUNT_KEY_FIELD_NAME);
+ }
+ StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey);
+ builder.credential(credential);
+ }
+
+ // Shared access signature
+ if (sharedAccessSignature != null) {
+ Optional<String> provided = getFirstNotNull(configuration, MANAGED_IDENTITY_ID_FIELD_NAME,
+ CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME,
+ CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ }
+ AzureSasCredential credential = new AzureSasCredential(sharedAccessSignature);
+ builder.credential(credential);
+ }
+
+ // Managed Identity auth
+ if (managedIdentityId != null) {
+ Optional<String> provided = getFirstNotNull(configuration, CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ MANAGED_IDENTITY_ID_FIELD_NAME);
+ }
+ builder.credential(new ManagedIdentityCredentialBuilder().clientId(managedIdentityId).build());
+ }
+
+ // Client secret & certificate auth
+ if (clientId != null) {
+ // Both (or neither) client secret and client secret were provided, only one is allowed
+ if ((clientSecret == null) == (clientCertificate == null)) {
+ if (clientSecret != null) {
+ throw new CompilationException(PARAMETERS_NOT_ALLOWED_AT_SAME_TIME, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME);
+ } else {
+ throw new CompilationException(REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT,
+ CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_ID_FIELD_NAME);
+ }
+ }
+
+ // Tenant ID is required
+ if (tenantId == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, TENANT_ID_FIELD_NAME,
+ CLIENT_ID_FIELD_NAME);
+ }
+
+ // Client certificate password is not allowed if client secret is used
+ if (clientCertificatePassword != null && clientSecret != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
+ CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, CLIENT_SECRET_FIELD_NAME);
+ }
+
+ // Use AD authentication
+ if (clientSecret != null) {
+ ClientSecretCredentialBuilder secret = new ClientSecretCredentialBuilder();
+ secret.clientId(clientId);
+ secret.tenantId(tenantId);
+ secret.clientSecret(clientSecret);
+ builder.credential(secret.build());
+ } else {
+ // Certificate
+ ClientCertificateCredentialBuilder certificate = new ClientCertificateCredentialBuilder();
+ certificate.clientId(clientId);
+ certificate.tenantId(tenantId);
+ try {
+ InputStream certificateContent = new ByteArrayInputStream(clientCertificate.getBytes(UTF_8));
+ if (clientCertificatePassword == null) {
+ Method pemCertificate = ClientCertificateCredentialBuilder.class
+ .getDeclaredMethod("pemCertificate", InputStream.class);
+ pemCertificate.setAccessible(true);
+ pemCertificate.invoke(certificate, certificateContent);
+ } else {
+ Method pemCertificate = ClientCertificateCredentialBuilder.class
+ .getDeclaredMethod("pfxCertificate", InputStream.class, String.class);
+ pemCertificate.setAccessible(true);
+ pemCertificate.invoke(certificate, certificateContent, clientCertificatePassword);
+ }
+ } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ex) {
+ throw new CompilationException(EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ }
+ builder.credential(certificate.build());
+ }
+ }
+
+ // If client id is not present, ensure client secret, certificate, tenant id and client certificate
+ // password are not present
+ if (clientId == null) {
+ Optional<String> provided = getFirstNotNull(configuration, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ }
+ }
+
+ try {
+ return builder.buildClient();
+ } catch (Exception ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ }
+
+ /**
+ * Builds the Azure data lake storage account using the provided configuration
+ *
+ * @param configuration properties
+ * @return client
+ */
+ public static DataLakeServiceClient buildAzureDatalakeClient(IApplicationContext appCtx,
+ Map<String, String> configuration) throws CompilationException {
+ String managedIdentityId = configuration.get(MANAGED_IDENTITY_ID_FIELD_NAME);
+ String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
+ String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
+ String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ String tenantId = configuration.get(TENANT_ID_FIELD_NAME);
+ String clientId = configuration.get(CLIENT_ID_FIELD_NAME);
+ String clientSecret = configuration.get(CLIENT_SECRET_FIELD_NAME);
+ String clientCertificate = configuration.get(CLIENT_CERTIFICATE_FIELD_NAME);
+ String clientCertificatePassword = configuration.get(CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME);
+ String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
+
+ // Client builder
+ DataLakeServiceClientBuilder builder = new DataLakeServiceClientBuilder();
+ int timeout = appCtx.getExternalProperties().getAzureRequestTimeout();
+ RequestRetryOptions requestRetryOptions = new RequestRetryOptions(null, null, timeout, null, null, null);
+ builder.retryOptions(requestRetryOptions);
+
+ // Endpoint is required
+ if (endpoint == null) {
+ throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME);
+ }
+ builder.endpoint(endpoint);
+
+ // Shared Key
+ if (accountName != null || accountKey != null) {
+ if (accountName == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_NAME_FIELD_NAME,
+ ACCOUNT_KEY_FIELD_NAME);
+ }
+
+ if (accountKey == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_KEY_FIELD_NAME,
+ ACCOUNT_NAME_FIELD_NAME);
+ }
+
+ Optional<String> provided = getFirstNotNull(configuration, SHARED_ACCESS_SIGNATURE_FIELD_NAME,
+ MANAGED_IDENTITY_ID_FIELD_NAME, CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ ACCOUNT_KEY_FIELD_NAME);
+ }
+ StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey);
+ builder.credential(credential);
+ }
+
+ // Shared access signature
+ if (sharedAccessSignature != null) {
+ Optional<String> provided = getFirstNotNull(configuration, MANAGED_IDENTITY_ID_FIELD_NAME,
+ CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME,
+ CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ }
+ AzureSasCredential credential = new AzureSasCredential(sharedAccessSignature);
+ builder.credential(credential);
+ }
+
+ // Managed Identity auth
+ if (managedIdentityId != null) {
+ Optional<String> provided = getFirstNotNull(configuration, CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ MANAGED_IDENTITY_ID_FIELD_NAME);
+ }
+ builder.credential(new ManagedIdentityCredentialBuilder().clientId(managedIdentityId).build());
+ }
+
+ // Client secret & certificate auth
+ if (clientId != null) {
+ // Both (or neither) client secret and client secret were provided, only one is allowed
+ if ((clientSecret == null) == (clientCertificate == null)) {
+ if (clientSecret != null) {
+ throw new CompilationException(PARAMETERS_NOT_ALLOWED_AT_SAME_TIME, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME);
+ } else {
+ throw new CompilationException(REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT,
+ CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_ID_FIELD_NAME);
+ }
+ }
+
+ // Tenant ID is required
+ if (tenantId == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, TENANT_ID_FIELD_NAME,
+ CLIENT_ID_FIELD_NAME);
+ }
+
+ // Client certificate password is not allowed if client secret is used
+ if (clientCertificatePassword != null && clientSecret != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
+ CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, CLIENT_SECRET_FIELD_NAME);
+ }
+
+ // Use AD authentication
+ if (clientSecret != null) {
+ ClientSecretCredentialBuilder secret = new ClientSecretCredentialBuilder();
+ secret.clientId(clientId);
+ secret.tenantId(tenantId);
+ secret.clientSecret(clientSecret);
+ builder.credential(secret.build());
+ } else {
+ // Certificate
+ ClientCertificateCredentialBuilder certificate = new ClientCertificateCredentialBuilder();
+ certificate.clientId(clientId);
+ certificate.tenantId(tenantId);
+ try {
+ InputStream certificateContent = new ByteArrayInputStream(clientCertificate.getBytes(UTF_8));
+ if (clientCertificatePassword == null) {
+ Method pemCertificate = ClientCertificateCredentialBuilder.class
+ .getDeclaredMethod("pemCertificate", InputStream.class);
+ pemCertificate.setAccessible(true);
+ pemCertificate.invoke(certificate, certificateContent);
+ } else {
+ Method pemCertificate = ClientCertificateCredentialBuilder.class
+ .getDeclaredMethod("pfxCertificate", InputStream.class, String.class);
+ pemCertificate.setAccessible(true);
+ pemCertificate.invoke(certificate, certificateContent, clientCertificatePassword);
+ }
+ } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ex) {
+ throw new CompilationException(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ builder.credential(certificate.build());
+ }
+ }
+
+ // If client id is not present, ensure client secret, certificate, tenant id and client certificate
+ // password are not present
+ if (clientId == null) {
+ Optional<String> provided = getFirstNotNull(configuration, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ }
+ }
+
+ try {
+ return builder.buildClient();
+ } catch (Exception ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ }
+
+ public static List<BlobItem> listBlobItems(BlobServiceClient blobServiceClient, Map<String, String> configuration,
+ AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
+ IWarningCollector warningCollector) throws CompilationException {
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+
+ List<BlobItem> filesOnly = new ArrayList<>();
+
+ // Ensure the validity of include/exclude
+ validateIncludeExclude(configuration);
+
+ BlobContainerClient blobContainer;
+ try {
+ blobContainer = blobServiceClient.getBlobContainerClient(container);
+
+ // Get all objects in a container and extract the paths to files
+ ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
+ listBlobsOptions.setPrefix(getPrefix(configuration));
+ Iterable<BlobItem> blobItems = blobContainer.listBlobs(listBlobsOptions, null);
+
+ // Collect the paths to files only
+ collectAndFilterBlobFiles(blobItems, includeExcludeMatcher.getPredicate(),
+ includeExcludeMatcher.getMatchersList(), filesOnly);
+
+ // Warn if no files are returned
+ if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
+ Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ warningCollector.warn(warning);
+ }
+ } catch (Exception ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+
+ return filesOnly;
+ }
+
+ /**
+ * Collects and filters the files only, and excludes any folders
+ *
+ * @param items storage items
+ * @param predicate predicate to test with for file filtration
+ * @param matchers include/exclude matchers to test against
+ * @param filesOnly List containing the files only (excluding folders)
+ */
+ private static void collectAndFilterBlobFiles(Iterable<BlobItem> items,
+ BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers, List<BlobItem> filesOnly) {
+ for (BlobItem item : items) {
+ String uri = item.getName();
+
+ // skip folders
+ if (uri.endsWith("/")) {
+ continue;
+ }
+
+ // No filter, add file
+ if (predicate.test(matchers, uri)) {
+ filesOnly.add(item);
+ }
+ }
+ }
+
+ public static List<PathItem> listDatalakePathItems(DataLakeServiceClient client, Map<String, String> configuration,
+ AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
+ IWarningCollector warningCollector) throws CompilationException {
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+
+ List<PathItem> filesOnly = new ArrayList<>();
+
+ // Ensure the validity of include/exclude
+ validateIncludeExclude(configuration);
+
+ DataLakeFileSystemClient fileSystemClient;
+ try {
+ fileSystemClient = client.getFileSystemClient(container);
+
+ // Get all objects in a container and extract the paths to files
+ ListPathsOptions listOptions = new ListPathsOptions();
+ boolean recursive = Boolean.parseBoolean(configuration.get(RECURSIVE_FIELD_NAME));
+ listOptions.setRecursive(recursive);
+ listOptions.setPath(getPrefix(configuration, false));
+ PagedIterable<PathItem> pathItems = fileSystemClient.listPaths(listOptions, null);
+
+ // Collect the paths to files only
+ collectAndFilterDatalakeFiles(pathItems, includeExcludeMatcher.getPredicate(),
+ includeExcludeMatcher.getMatchersList(), filesOnly);
+
+ // Warn if no files are returned
+ if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
+ Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ warningCollector.warn(warning);
+ }
+ } catch (Exception ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+
+ return filesOnly;
+ }
+
+ /**
+ * Collects and filters the files only, and excludes any folders
+ *
+ * @param items storage items
+ * @param predicate predicate to test with for file filtration
+ * @param matchers include/exclude matchers to test against
+ * @param filesOnly List containing the files only (excluding folders)
+ */
+ private static void collectAndFilterDatalakeFiles(Iterable<PathItem> items,
+ BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers, List<PathItem> filesOnly) {
+ for (PathItem item : items) {
+ String uri = item.getName();
+
+ // skip folders
+ if (uri.endsWith("/")) {
+ continue;
+ }
+
+ // No filter, add file
+ if (predicate.test(matchers, uri)) {
+ filesOnly.add(item);
+ }
+ }
+ }
+
+ /**
+ * Validate external dataset properties
+ *
+ * @param configuration properties
+ * @throws CompilationException Compilation exception
+ */
+ public static void validateAzureBlobProperties(Map<String, String> configuration, SourceLocation srcLoc,
+ IWarningCollector collector, IApplicationContext appCtx) throws CompilationException {
+
+ // check if the format property is present
+ if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+ throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
+ }
+
+ validateIncludeExclude(configuration);
+
+ // Check if the bucket is present
+ BlobServiceClient blobServiceClient;
+ try {
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+ blobServiceClient = buildAzureBlobClient(appCtx, configuration);
+ BlobContainerClient blobContainer = blobServiceClient.getBlobContainerClient(container);
+
+ // Get all objects in a container and extract the paths to files
+ ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
+ listBlobsOptions.setPrefix(getPrefix(configuration));
+ Iterable<BlobItem> blobItems = blobContainer.listBlobs(listBlobsOptions, null);
+
+ if (!blobItems.iterator().hasNext() && collector.shouldWarn()) {
+ Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ collector.warn(warning);
+ }
+ } catch (CompilationException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ }
+
+ /**
+ * Validate external dataset properties
+ *
+ * @param configuration properties
+ * @throws CompilationException Compilation exception
+ */
+ public static void validateAzureDataLakeProperties(Map<String, String> configuration, SourceLocation srcLoc,
+ IWarningCollector collector, IApplicationContext appCtx) throws CompilationException {
+
+ // check if the format property is present
+ if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+ throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
+ }
+
+ validateIncludeExclude(configuration);
+
+ // Check if the bucket is present
+ DataLakeServiceClient dataLakeServiceClient;
+ try {
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+ dataLakeServiceClient = buildAzureDatalakeClient(appCtx, configuration);
+ DataLakeFileSystemClient fileSystemClient = dataLakeServiceClient.getFileSystemClient(container);
+
+ // Get all objects in a container and extract the paths to files
+ ListPathsOptions listPathsOptions = new ListPathsOptions();
+ listPathsOptions.setPath(getPrefix(configuration));
+ Iterable<PathItem> blobItems = fileSystemClient.listPaths(listPathsOptions, null);
+
+ if (!blobItems.iterator().hasNext() && collector.shouldWarn()) {
+ Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ collector.warn(warning);
+ }
+ } catch (CompilationException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ }
+
+ /**
+ * Builds the Azure Blob storage client using the provided configuration
+ *
+ * @param configuration properties
+ * @see <a href="https://docs.microsoft.com/en-us/azure/databricks/data/data-sources/azure/azure-storage">Azure
+ * Blob storage</a>
+ */
+ public static void configureAzureHdfsJobConf(JobConf conf, Map<String, String> configuration, String endPoint) {
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+ String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
+ String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+
+ //Disable caching S3 FileSystem
+ HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_AZURE_BLOB_PROTOCOL);
+
+ //Key for Hadoop configuration
+ StringBuilder hadoopKey = new StringBuilder();
+ //Value for Hadoop configuration
+ String hadoopValue;
+ if (accountKey != null || sharedAccessSignature != null) {
+ if (accountKey != null) {
+ hadoopKey.append(HADOOP_AZURE_FS_ACCOUNT_KEY).append('.');
+ //Set only the AccountKey
+ hadoopValue = accountKey;
+ } else {
+ //Use SAS for Hadoop FS as connectionString is provided
+ hadoopKey.append(HADOOP_AZURE_FS_SAS).append('.');
+ //Setting the container is required for SAS
+ hadoopKey.append(container).append('.');
+ //Set the connection string for SAS
+ hadoopValue = sharedAccessSignature;
+ }
+ //Set the endPoint, which includes the AccountName
+ hadoopKey.append(endPoint);
+ //Tells Hadoop we are reading from Blob Storage
+ conf.set(hadoopKey.toString(), hadoopValue);
+ }
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
new file mode 100644
index 0000000..f2dbde7
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.google.gcs;
+
+public class GCSConstants {
+ private GCSConstants() {
+ throw new AssertionError("do not instantiate");
+ }
+
+ public static final String APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME = "applicationDefaultCredentials";
+ public static final String JSON_CREDENTIALS_FIELD_NAME = "jsonCredentials";
+ public static final String ENDPOINT_FIELD_NAME = "endpoint";
+
+ /*
+ * Hadoop internal configuration
+ */
+ public static final String HADOOP_GCS_PROTOCOL = "gs";
+
+ // hadoop credentials
+ public static final String HADOOP_AUTH_TYPE = "fs.gs.auth.type";
+ public static final String HADOOP_AUTH_UNAUTHENTICATED = "UNAUTHENTICATED";
+ public static final String HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE = "SERVICE_ACCOUNT_JSON_KEYFILE";
+ public static final String HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH =
+ "google.cloud.auth.service.account.json.keyfile";
+
+ // gs hadoop parameters
+ public static final String HADOOP_SUPPORT_COMPRESSED = "fs.gs.inputstream.support.gzip.encoding.enable";
+ public static final String HADOOP_ENDPOINT = "fs.gs.storage.root.url";
+ public static final String HADOOP_MAX_REQUESTS_PER_BATCH = "fs.gs.max.requests.per.batch";
+ public static final String HADOOP_BATCH_THREADS = "fs.gs.batch.threads";
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
new file mode 100644
index 0000000..3efb041
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.google.gcs;
+
+import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
+import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.ENDPOINT_FIELD_NAME;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_TYPE;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_UNAUTHENTICATED;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_ENDPOINT;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_GCS_PROTOCOL;
+import static org.apache.asterix.external.util.google.gcs.GCSConstants.JSON_CREDENTIALS_FIELD_NAME;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiPredicate;
+import java.util.regex.Matcher;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory.IncludeExcludeMatcher;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.HDFSUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.exceptions.Warning;
+
+import com.google.api.gax.paging.Page;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+
+public class GCSUtils {
+ private GCSUtils() {
+ throw new AssertionError("do not instantiate");
+
+ }
+
+ /**
+ * Builds the client using the provided configuration
+ *
+ * @param configuration properties
+ * @return clientasterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+ * @throws CompilationException CompilationException
+ */
+ public static Storage buildClient(Map<String, String> configuration) throws CompilationException {
+ String applicationDefaultCredentials = configuration.get(APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
+ String jsonCredentials = configuration.get(JSON_CREDENTIALS_FIELD_NAME);
+ String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
+
+ StorageOptions.Builder builder = StorageOptions.newBuilder();
+
+ // default credentials provider
+ if (applicationDefaultCredentials != null) {
+ // only "true" value is allowed
+ if (!applicationDefaultCredentials.equalsIgnoreCase("true")) {
+ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE,
+ APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME, "true");
+ }
+
+ // no other authentication parameters are allowed
+ if (jsonCredentials != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, JSON_CREDENTIALS_FIELD_NAME,
+ APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
+ }
+
+ try {
+ builder.setCredentials(GoogleCredentials.getApplicationDefault());
+ } catch (IOException ex) {
+ throw CompilationException.create(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ }
+
+ // json credentials
+ if (jsonCredentials != null) {
+ try (InputStream credentialsStream = new ByteArrayInputStream(jsonCredentials.getBytes())) {
+ builder.setCredentials(GoogleCredentials.fromStream(credentialsStream));
+ } catch (IOException ex) {
+ throw new CompilationException(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ }
+
+ if (endpoint != null) {
+ builder.setHost(endpoint);
+ }
+
+ return builder.build().getService();
+ }
+
+ /**
+ * Validate external dataset properties
+ *
+ * @param configuration properties
+ * @throws CompilationException Compilation exception
+ */
+ public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc,
+ IWarningCollector collector) throws CompilationException {
+
+ // check if the format property is present
+ if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+ throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
+ }
+
+ validateIncludeExclude(configuration);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+
+ try {
+ Storage.BlobListOption limitOption = Storage.BlobListOption.pageSize(1);
+ Storage.BlobListOption prefixOption = Storage.BlobListOption.prefix(getPrefix(configuration));
+ Storage storage = buildClient(configuration);
+ Page<Blob> items = storage.list(container, limitOption, prefixOption);
+
+ if (!items.iterateAll().iterator().hasNext() && collector.shouldWarn()) {
+ Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ collector.warn(warning);
+ }
+ } catch (CompilationException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+ }
+
+ public static List<Blob> listItems(Map<String, String> configuration, IncludeExcludeMatcher includeExcludeMatcher,
+ IWarningCollector warningCollector) throws CompilationException {
+ // Prepare to retrieve the objects
+ List<Blob> filesOnly = new ArrayList<>();
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+ Storage gcs = buildClient(configuration);
+ Storage.BlobListOption options = Storage.BlobListOption.prefix(ExternalDataUtils.getPrefix(configuration));
+ Page<Blob> items;
+
+ try {
+ items = gcs.list(container, options);
+ } catch (BaseServiceException ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
+ }
+
+ // Collect the paths to files only
+ collectAndFilterFiles(items, includeExcludeMatcher.getPredicate(), includeExcludeMatcher.getMatchersList(),
+ filesOnly);
+
+ // Warn if no files are returned
+ if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
+ Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ warningCollector.warn(warning);
+ }
+
+ return filesOnly;
+ }
+
+ /**
+ * Excludes paths ending with "/" as that's a directory indicator, we need to return the files only
+ *
+ * @param items List of returned objects
+ */
+ private static void collectAndFilterFiles(Page<Blob> items, BiPredicate<List<Matcher>, String> predicate,
+ List<Matcher> matchers, List<Blob> filesOnly) {
+ for (Blob item : items.iterateAll()) {
+ // skip folders
+ if (item.getName().endsWith("/")) {
+ continue;
+ }
+
+ // No filter, add file
+ if (predicate.test(matchers, item.getName())) {
+ filesOnly.add(item);
+ }
+ }
+ }
+
+ /**
+ * Builds the client using the provided configuration
+ *
+ * @param configuration properties
+ * @param numberOfPartitions number of partitions in the cluster
+ */
+ public static void configureHdfsJobConf(JobConf conf, Map<String, String> configuration, int numberOfPartitions) {
+ String jsonCredentials = configuration.get(JSON_CREDENTIALS_FIELD_NAME);
+ String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
+
+ // disable caching FileSystem
+ HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_GCS_PROTOCOL);
+
+ // TODO(htowaileb): needs further testing, recommended to disable by gcs-hadoop team
+ conf.set(GCSConstants.HADOOP_SUPPORT_COMPRESSED, ExternalDataConstants.FALSE);
+
+ // TODO(htowaileb): needs further testing
+ // set number of threads
+ // conf.set(GCSConstants.HADOOP_MAX_REQUESTS_PER_BATCH, String.valueOf(numberOfPartitions));
+ // conf.set(GCSConstants.HADOOP_BATCH_THREADS, String.valueOf(numberOfPartitions));
+
+ // authentication method
+ // TODO(htowaileb): find a way to pass the content instead of the path to keyfile, this line is temporary
+ Path credentials = Path.of("credentials.json");
+ if (jsonCredentials == null) {
+ // anonymous access
+ conf.set(HADOOP_AUTH_TYPE, HADOOP_AUTH_UNAUTHENTICATED);
+ } else {
+ // TODO(htowaileb) need to pass the file content
+ conf.set(HADOOP_AUTH_TYPE, HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE);
+ conf.set(HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH, credentials.toAbsolutePath().toString());
+ }
+
+ // set endpoint if provided, default is https://storage.googleapis.com/
+ if (endpoint != null) {
+ conf.set(HADOOP_ENDPOINT, endpoint);
+ }
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/asterixdb/asterix-external-data/src/main/java/org/checkerframework/checker/nullness/compatqual/NullableDecl.java
similarity index 61%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to asterixdb/asterix-external-data/src/main/java/org/checkerframework/checker/nullness/compatqual/NullableDecl.java
index 3c1a24d..a4cb401 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/checkerframework/checker/nullness/compatqual/NullableDecl.java
@@ -16,20 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.checkerframework.checker.nullness.compatqual;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/*
+ * This is a clean room implementation of the NullableDecl interface based
+ * on 3 requirements:
+ * 1. shall be an @interface named NullableDecl
+ * 2. shall be in the org.checkerframework.checker.nullness.compatqual package
+ * 3. the rention policy for the interface shall be RUNTIME
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+@Retention(RetentionPolicy.RUNTIME)
+public @interface NullableDecl {
}
diff --git a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
index dceed82..88f3fcb 100644
--- a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
+++ b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
@@ -26,4 +26,5 @@
org.apache.asterix.external.input.record.reader.azure.blob.AzureBlobReaderFactory
org.apache.asterix.external.input.record.reader.azure.datalake.AzureDataLakeReaderFactory
org.apache.asterix.external.input.record.reader.azure.parquet.AzureBlobParquetReaderFactory
-org.apache.asterix.external.input.record.reader.azure.parquet.AzureDataLakeParquetReaderFactory
\ No newline at end of file
+org.apache.asterix.external.input.record.reader.azure.parquet.AzureDataLakeParquetReaderFactory
+org.apache.asterix.external.input.record.reader.gcs.parquet.GCSParquetReaderFactory
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
index 90ea04b..91afbd8 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
@@ -18,8 +18,8 @@
*/
package org.apache.asterix.external.input.record.reader.awss3;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_INTERNAL_ERROR;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_SLOW_DOWN;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
diff --git a/asterixdb/asterix-fuzzyjoin/pom.xml b/asterixdb/asterix-fuzzyjoin/pom.xml
index 65ffa19..1cc32ee 100644
--- a/asterixdb/asterix-fuzzyjoin/pom.xml
+++ b/asterixdb/asterix-fuzzyjoin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-fuzzyjoin</artifactId>
diff --git a/asterixdb/asterix-geo/pom.xml b/asterixdb/asterix-geo/pom.xml
index 667db8e..cdf9d56 100644
--- a/asterixdb/asterix-geo/pom.xml
+++ b/asterixdb/asterix-geo/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-geo</artifactId>
diff --git a/asterixdb/asterix-lang-common/pom.xml b/asterixdb/asterix-lang-common/pom.xml
index e58125c..45dacad 100644
--- a/asterixdb/asterix-lang-common/pom.xml
+++ b/asterixdb/asterix-lang-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/AbstractExtensionClause.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/AbstractExtensionClause.java
new file mode 100644
index 0000000..af9015b
--- /dev/null
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/AbstractExtensionClause.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.base;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public abstract class AbstractExtensionClause extends AbstractClause {
+ /**
+ * Parent languages should handle extension functionality via a new method in {@link IVisitorExtension}.
+ */
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws CompilationException {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, getSourceLocation(),
+ "Unhandled dispatch to an extension clause node!");
+ }
+
+ public abstract IVisitorExtension getVisitorExtension();
+
+ @Override
+ public final ClauseType getClauseType() {
+ return ClauseType.EXTENSION;
+ }
+}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Clause.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Clause.java
index 14f836b..95ed6df 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Clause.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Clause.java
@@ -43,7 +43,9 @@
SELECT_ELEMENT,
SELECT_REGULAR,
SELECT_SET_OPERATION,
- UNNEST_CLAUSE
+ UNNEST_CLAUSE,
+
+ EXTENSION
}
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IVisitorExtension.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IVisitorExtension.java
new file mode 100644
index 0000000..9007eb8
--- /dev/null
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IVisitorExtension.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.base;
+
+import java.util.Collection;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.lang.common.expression.AbstractCallExpression;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.parser.ScopeChecker;
+import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+/**
+ * Contract for all extension AST nodes to allow existing rewrites to support language extensions.
+ */
+public interface IVisitorExtension {
+ Expression simpleExpressionDispatch(ILangVisitor<Expression, ILangExpression> simpleExpressionVisitor,
+ ILangExpression argument) throws CompilationException;
+
+ Void freeVariableDispatch(ILangVisitor<Void, Collection<VariableExpr>> freeVariableVisitor,
+ Collection<VariableExpr> freeVariables) throws CompilationException;
+
+ Void bindingVariableDispatch(ILangVisitor<Void, Collection<VariableExpr>> bindingVariableVisitor,
+ Collection<VariableExpr> bindingVariables) throws CompilationException;
+
+ Expression variableScopeDispatch(ILangVisitor<Expression, ILangExpression> scopingVisitor, ILangExpression argument,
+ ScopeChecker scopeChecker) throws CompilationException;
+
+ ILangExpression deepCopyDispatch(ILangVisitor<ILangExpression, Void> deepCopyVisitor) throws CompilationException;
+
+ Pair<ILangExpression, VariableSubstitutionEnvironment> remapCloneDispatch(
+ ILangVisitor<Pair<ILangExpression, VariableSubstitutionEnvironment>, VariableSubstitutionEnvironment> remapCloneVisitor,
+ VariableSubstitutionEnvironment substitutionEnvironment) throws CompilationException;
+
+ Boolean inlineUDFsDispatch(ILangVisitor<Boolean, Void> inlineUDFsVisitor) throws CompilationException;
+
+ Void gatherFunctionsDispatch(ILangVisitor<Void, Void> gatherFunctionsVisitor,
+ Collection<? super AbstractCallExpression> functionCalls) throws CompilationException;
+
+ Boolean checkSubqueryDispatch(ILangVisitor<Boolean, ILangExpression> checkSubqueryVisitor, ILangExpression argument)
+ throws CompilationException;
+
+ Boolean check92AggregateDispatch(ILangVisitor<Boolean, ILangExpression> check92AggregateVisitor,
+ ILangExpression argument) throws CompilationException;
+
+ Boolean checkNonFunctionalDispatch(ILangVisitor<Boolean, Void> checkNonFunctionalVisitor)
+ throws CompilationException;
+
+ Boolean checkDatasetOnlyDispatch(ILangVisitor<Boolean, VariableExpr> checkDatasetOnlyVisitor,
+ VariableExpr datasetCandidate) throws CompilationException;
+}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
index 4330c4e..31a9c51 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
@@ -81,7 +81,6 @@
SET,
TYPE_DECL,
TYPE_DROP,
- WRITE,
CREATE_INDEX,
CREATE_DATAVERSE,
CREATE_VIEW,
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/rewrites/LangRewritingContext.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/rewrites/LangRewritingContext.java
index 95cccb0..19b3cfa 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/rewrites/LangRewritingContext.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/rewrites/LangRewritingContext.java
@@ -32,7 +32,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.Counter;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-public final class LangRewritingContext {
+public class LangRewritingContext {
private final MetadataProvider metadataProvider;
private final IWarningCollector warningCollector;
private final Map<FunctionSignature, FunctionDecl> declaredFunctions;
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
index 942e39b..8f48db0 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
@@ -20,6 +20,7 @@
import java.util.Map;
+import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.metadata.DataverseName;
@@ -31,6 +32,7 @@
import org.apache.asterix.lang.common.util.ConfigurationUtil;
import org.apache.asterix.lang.common.util.DatasetDeclParametersUtil;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.object.base.AdmObjectNode;
import org.apache.asterix.object.base.IAdmNode;
import org.apache.asterix.runtime.compression.CompressionManager;
@@ -43,7 +45,7 @@
protected final DatasetType datasetType;
protected final IDatasetDetailsDecl datasetDetailsDecl;
protected final Map<String, String> hints;
- private AdmObjectNode withObjectNode;
+ private final AdmObjectNode withObjectNode;
protected final boolean ifNotExists;
public DatasetDecl(DataverseName dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType,
@@ -132,6 +134,23 @@
.getOptionalString(DatasetDeclParametersUtil.STORAGE_BLOCK_COMPRESSION_SCHEME_PARAMETER_NAME);
}
+ public DatasetFormatInfo getDatasetFormatInfo(int defaultMaxTupleCount, float defaultFreeSpaceTolerance) {
+ final AdmObjectNode datasetFormatNode =
+ (AdmObjectNode) withObjectNode.get(DatasetDeclParametersUtil.DATASET_FORMAT_PARAMETER_NAME);
+ if (datasetType != DatasetType.INTERNAL || datasetFormatNode == null) {
+ return DatasetFormatInfo.DEFAULT;
+ }
+ DatasetConfig.DatasetFormat datasetFormat = DatasetConfig.DatasetFormat.getFormat(
+ datasetFormatNode.getOptionalString(DatasetDeclParametersUtil.DATASET_FORMAT_FORMAT_PARAMETER_NAME));
+ int maxTupleCount = datasetFormatNode.getOptionalInt(
+ DatasetDeclParametersUtil.DATASET_FORMAT_MAX_TUPLE_COUNT_PARAMETER_NAME, defaultMaxTupleCount);
+ float freeSpaceTolerance = datasetFormatNode.getOptionalFloat(
+ DatasetDeclParametersUtil.DATASET_FORMAT_FREE_SPACE_TOLERANCE_PARAMETER_NAME,
+ defaultFreeSpaceTolerance);
+
+ return new DatasetFormatInfo(datasetFormat, maxTupleCount, freeSpaceTolerance);
+ }
+
public Map<String, String> getHints() {
return hints;
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java
deleted file mode 100644
index d4c11e4..0000000
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.lang.common.statement;
-
-import org.apache.asterix.common.exceptions.CompilationException;
-import org.apache.asterix.lang.common.base.AbstractStatement;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.struct.Identifier;
-import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
-
-public class WriteStatement extends AbstractStatement {
-
- private final Identifier ncName;
- private final String fileName;
- private final String writerClassName;
-
- public WriteStatement(Identifier ncName, String fileName, String writerClassName) {
- this.ncName = ncName;
- this.fileName = fileName;
- this.writerClassName = writerClassName;
- }
-
- public Identifier getNcName() {
- return ncName;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public String getWriterClassName() {
- return writerClassName;
- }
-
- @Override
- public Kind getKind() {
- return Statement.Kind.WRITE;
- }
-
- @Override
- public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws CompilationException {
- return visitor.visit(this, arg);
- }
-
- @Override
- public byte getCategory() {
- return Category.PROCEDURE;
- }
-
-}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
index 0aef69f..882f61c 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
@@ -62,6 +62,16 @@
*/
public static final String NODE_GROUP_NAME = "node-group";
public static final String NODE_GROUP_NAME_PARAMETER_NAME = "name";
+
+ /* ***********************************************
+ * Dataset Format Type
+ * ***********************************************
+ */
+ public static final String DATASET_FORMAT_PARAMETER_NAME = "dataset-format";
+ public static final String DATASET_FORMAT_FORMAT_PARAMETER_NAME = "format";
+ public static final String DATASET_FORMAT_MAX_TUPLE_COUNT_PARAMETER_NAME = "max-tuple-count";
+ public static final String DATASET_FORMAT_FREE_SPACE_TOLERANCE_PARAMETER_NAME = "free-space-tolerance";
+
/* ***********************************************
* Private members
* ***********************************************
@@ -90,11 +100,12 @@
}
private static ARecordType getWithObjectType() {
- final String[] withNames =
- { MERGE_POLICY_PARAMETER_NAME, STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME, NODE_GROUP_NAME };
+ final String[] withNames = { MERGE_POLICY_PARAMETER_NAME, STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME,
+ NODE_GROUP_NAME, DATASET_FORMAT_PARAMETER_NAME };
final IAType[] withTypes = { AUnionType.createUnknownableType(getMergePolicyType()),
AUnionType.createUnknownableType(getStorageBlockCompressionType()),
- AUnionType.createUnknownableType(getNodeGroupType()) };
+ AUnionType.createUnknownableType(getNodeGroupType()),
+ AUnionType.createUnknownableType(getDatasetFormatType()) };
return new ARecordType("withObject", withNames, withTypes, false);
}
@@ -134,8 +145,8 @@
}
/**
- * Adjusts dataset inline type definition if it has primary key specification:
- * forces NOT UNKNOWN on fields that are part of primary key.
+ * Adjusts dataset inline type definition if it has primary key specification:
+ * forces NOT UNKNOWN on fields that are part of primary key.
*/
public static void adjustInlineTypeDecl(TypeExpression typeDecl, List<List<String>> primaryKeyFields,
List<Integer> primaryKeySources, boolean isMeta) {
@@ -166,4 +177,12 @@
throw new IllegalStateException(typeDecl.getTypeKind().toString());
}
}
+
+ private static ARecordType getDatasetFormatType() {
+ final String[] formatFieldNames = { DATASET_FORMAT_FORMAT_PARAMETER_NAME,
+ DATASET_FORMAT_MAX_TUPLE_COUNT_PARAMETER_NAME, DATASET_FORMAT_FREE_SPACE_TOLERANCE_PARAMETER_NAME };
+ final IAType[] formatFieldTypes = { BuiltinType.ASTRING, AUnionType.createUnknownableType(BuiltinType.AINT64),
+ AUnionType.createUnknownableType(BuiltinType.ADOUBLE) };
+ return new ARecordType(DATASET_FORMAT_PARAMETER_NAME, formatFieldNames, formatFieldTypes, false);
+ }
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
index 52775d3..077748d 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
@@ -31,6 +31,7 @@
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Expression.Kind;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -264,6 +265,11 @@
}
@Override
+ public Boolean visit(IVisitorExtension ve, Void arg) throws CompilationException {
+ return ve.inlineUDFsDispatch(this);
+ }
+
+ @Override
public Boolean visit(InsertStatement insert, Void arg) throws CompilationException {
boolean changed = false;
Expression returnExpression = insert.getReturnExpression();
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java
index d73c264..06f22b7 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java
@@ -29,6 +29,7 @@
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Expression.Kind;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -339,6 +340,12 @@
}
@Override
+ public Pair<ILangExpression, VariableSubstitutionEnvironment> visit(IVisitorExtension ve,
+ VariableSubstitutionEnvironment arg) throws CompilationException {
+ return ve.remapCloneDispatch(this, arg);
+ }
+
+ @Override
public Pair<ILangExpression, VariableSubstitutionEnvironment> visit(FieldAccessor fa,
VariableSubstitutionEnvironment env) throws CompilationException {
Pair<ILangExpression, VariableSubstitutionEnvironment> p = fa.getExpr().accept(this, env);
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index f42f2f0..7424398 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -110,7 +110,6 @@
import org.apache.asterix.lang.common.statement.UpdateStatement;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.statement.ViewDropStatement;
-import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
@@ -510,16 +509,6 @@
}
@Override
- public Void visit(WriteStatement ws, Integer step) throws CompilationException {
- out.print(skip(step) + "write output to " + ws.getNcName() + ":" + revertStringToQuoted(ws.getFileName()));
- if (ws.getWriterClassName() != null) {
- out.print(" using " + ws.getWriterClassName());
- }
- out.println();
- return null;
- }
-
- @Override
public Void visit(SetStatement ss, Integer step) throws CompilationException {
out.println(skip(step) + "set " + revertStringToQuoted(ss.getPropName()) + " "
+ revertStringToQuoted(ss.getPropValue()) + ";\n");
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
index b7cf7af..6dcfb83 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
@@ -25,6 +25,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -227,6 +228,11 @@
}
@Override
+ public Void visit(IVisitorExtension ve, Void arg) throws CompilationException {
+ return ve.gatherFunctionsDispatch(this, calls);
+ }
+
+ @Override
public Void visit(WhereClause wc, Void arg) throws CompilationException {
wc.getWhereExpr().accept(this, arg);
return null;
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
index 79e90a4..aa875b4 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
@@ -60,7 +60,6 @@
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.statement.SetStatement;
import org.apache.asterix.lang.common.statement.TypeDecl;
-import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.visitor.base.AbstractQueryExpressionVisitor;
@@ -420,16 +419,6 @@
}
@Override
- public Void visit(WriteStatement ws, Integer step) throws CompilationException {
- out.print(skip(step) + "WriteOutputTo " + ws.getNcName() + ":" + ws.getFileName());
- if (ws.getWriterClassName() != null) {
- out.print(" using " + ws.getWriterClassName());
- }
- out.println();
- return null;
- }
-
- @Override
public Void visit(SetStatement ss, Integer step) throws CompilationException {
out.println(skip(step) + "Set " + ss.getPropName() + "=" + ss.getPropValue());
return null;
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractAstVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractAstVisitor.java
index a422ef1..a6092c6 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractAstVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractAstVisitor.java
@@ -19,6 +19,8 @@
package org.apache.asterix.lang.common.visitor.base;
import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -130,4 +132,8 @@
return null;
}
+ @Override
+ public R visit(IVisitorExtension ve, T arg) throws CompilationException {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, "Extension dispatch not implemented!");
+ }
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
index a060d1e..a7444e9 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
@@ -68,7 +68,6 @@
import org.apache.asterix.lang.common.statement.UpdateStatement;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.statement.ViewDropStatement;
-import org.apache.asterix.lang.common.statement.WriteStatement;
public abstract class AbstractQueryExpressionVisitor<R, T> implements ILangVisitor<R, T> {
@@ -163,11 +162,6 @@
}
@Override
- public R visit(WriteStatement ws, T arg) throws CompilationException {
- return null;
- }
-
- @Override
public R visit(CreateDataverseStatement del, T arg) throws CompilationException {
return null;
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
index b9de347..541567d 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
@@ -19,6 +19,7 @@
package org.apache.asterix.lang.common.visitor.base;
import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -85,7 +86,6 @@
import org.apache.asterix.lang.common.statement.UpdateStatement;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.statement.ViewDropStatement;
-import org.apache.asterix.lang.common.statement.WriteStatement;
public interface ILangVisitor<R, T> {
@@ -169,8 +169,6 @@
R visit(TypeDropStatement del, T arg) throws CompilationException;
- R visit(WriteStatement ws, T arg) throws CompilationException;
-
R visit(SetStatement ss, T arg) throws CompilationException;
R visit(DisconnectFeedStatement del, T arg) throws CompilationException;
@@ -222,4 +220,6 @@
R visit(ViewDropStatement vds, T arg) throws CompilationException;
R visit(ViewDecl vd, T arg) throws CompilationException;
+
+ R visit(IVisitorExtension ve, T arg) throws CompilationException;
}
diff --git a/asterixdb/asterix-lang-sqlpp/pom.xml b/asterixdb/asterix-lang-sqlpp/pom.xml
index 0248ed7..1e7882b 100644
--- a/asterixdb/asterix-lang-sqlpp/pom.xml
+++ b/asterixdb/asterix-lang-sqlpp/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/clause/SelectClause.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/clause/SelectClause.java
index ea6bac9..c902dbe 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/clause/SelectClause.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/clause/SelectClause.java
@@ -19,23 +19,43 @@
package org.apache.asterix.lang.sqlpp.clause;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
public class SelectClause extends AbstractClause {
-
+ private final List<List<String>> fieldExclusions = new ArrayList<>();
private SelectElement selectElement;
private SelectRegular selectRegular;
private boolean distinct;
- public SelectClause(SelectElement selectElement, SelectRegular selectRegular, boolean distinct) {
+ public SelectClause(SelectElement selectElement, SelectRegular selectRegular, List<List<String>> fieldExclusions,
+ boolean distinct) {
+ if (selectElement != null && selectRegular != null) {
+ throw new IllegalArgumentException("SELECT-ELEMENT and SELECT-REGULAR cannot both be specified.");
+ }
+ if (selectElement != null && fieldExclusions != null && !fieldExclusions.isEmpty()) {
+ throw new IllegalArgumentException("SELECT-ELEMENT and EXCLUDE cannot both be specified.");
+ }
+
this.selectElement = selectElement;
this.selectRegular = selectRegular;
this.distinct = distinct;
+ if (fieldExclusions != null) {
+ this.fieldExclusions.addAll(fieldExclusions);
+ }
+ }
+
+ public SelectClause(SelectElement selectElement, SelectRegular selectRegular, boolean distinct) {
+ this(selectElement, selectRegular, null, distinct);
}
@Override
@@ -48,7 +68,12 @@
return ClauseType.SELECT_CLAUSE;
}
- public void setSelectElement(SelectElement selectElement) {
+ public void setSelectElement(SelectElement selectElement) throws CompilationException {
+ if (!fieldExclusions.isEmpty() && selectElement != null) {
+ // We forbid SELECT VALUE and EXCLUDE at the parser, so we should never reach here.
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, getSourceLocation(),
+ "SELECT ELEMENT and EXCLUDE cannot coexist!");
+ }
this.selectElement = selectElement;
this.selectRegular = null;
}
@@ -82,15 +107,25 @@
this.distinct = distinct;
}
+ public List<List<String>> getFieldExclusions() {
+ return fieldExclusions;
+ }
+
@Override
public String toString() {
- return "select " + (distinct ? "distinct " : "")
- + (selectElement() ? "element " + selectElement : String.valueOf(selectRegular));
+ String distinctString = distinct ? "distinct " : "";
+ String valueString = selectElement() ? ("element " + selectElement) : String.valueOf(selectRegular);
+ String exceptString = "";
+ if (!fieldExclusions.isEmpty()) {
+ final Function<List<String>, String> fieldBuilder = f -> String.join(".", f);
+ exceptString = " exclude " + fieldExclusions.stream().map(fieldBuilder).collect(Collectors.joining(", "));
+ }
+ return String.format("select %s%s%s", distinctString, valueString, exceptString);
}
@Override
public int hashCode() {
- return Objects.hash(distinct, selectElement, selectRegular);
+ return Objects.hash(distinct, selectElement, selectRegular, fieldExclusions);
}
@Override
@@ -103,6 +138,7 @@
}
SelectClause target = (SelectClause) object;
return distinct == target.distinct && Objects.equals(selectElement, target.selectElement)
- && Objects.equals(selectRegular, target.selectRegular);
+ && Objects.equals(selectRegular, target.selectRegular)
+ && Objects.equals(fieldExclusions, target.fieldExclusions);
}
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
index 557f17d..32549d9 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
@@ -61,7 +61,7 @@
/**
* This rewriter is used to rewrite body expression of user defined functions and views
*/
-class SqlppFunctionBodyRewriter extends SqlppQueryRewriter {
+public class SqlppFunctionBodyRewriter extends SqlppQueryRewriter {
public SqlppFunctionBodyRewriter(IParserFactory parserFactory) {
super(parserFactory);
@@ -97,6 +97,9 @@
// Inlines column aliases.
inlineColumnAlias();
+ // Rewrite SELECT EXCLUDE to use OBJECT_REMOVE_FIELDS.
+ rewriteSelectExcludeSugar();
+
// Window expression core rewrites.
rewriteWindowExpressions();
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index 2f4fcc8..38177d8 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -58,6 +58,7 @@
import org.apache.asterix.lang.sqlpp.rewrites.visitor.InlineColumnAliasVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.InlineWithExpressionVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.OperatorExpressionVisitor;
+import org.apache.asterix.lang.sqlpp.rewrites.visitor.SelectExcludeRewriteSugarVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SetOperationVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlCompatRewriteVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppCaseAggregateExtractionVisitor;
@@ -162,6 +163,9 @@
// Inlines column aliases.
inlineColumnAlias();
+ // Rewrite SELECT EXCLUDE to use OBJECT_REMOVE_FIELDS.
+ rewriteSelectExcludeSugar();
+
// Window expression core rewrites.
rewriteWindowExpressions();
@@ -350,6 +354,12 @@
}
}
+ protected void rewriteSelectExcludeSugar() throws CompilationException {
+ SelectExcludeRewriteSugarVisitor selectExcludeRewriteSugarVisitor =
+ new SelectExcludeRewriteSugarVisitor(context);
+ rewriteTopExpr(selectExcludeRewriteSugarVisitor, null);
+ }
+
private <R, T> R rewriteTopExpr(ILangVisitor<R, T> visitor, T arg) throws CompilationException {
R result = topStatement.accept(visitor, arg);
logExpression(">>>> AST After", visitor.getClass().getSimpleName());
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SelectExcludeRewriteSugarVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SelectExcludeRewriteSugarVisitor.java
new file mode 100644
index 0000000..e900c03
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SelectExcludeRewriteSugarVisitor.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.context.Scope;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.clause.Projection;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.clause.SelectElement;
+import org.apache.asterix.lang.sqlpp.clause.SelectRegular;
+import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
+import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
+import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+/**
+ * Rewrites the exclusion list of a SELECT clause into a OBJECT_REMOVE_FIELDS function application. This rewrite
+ * <b>MUST</b> run after {@link InlineColumnAliasVisitor}.
+ * <p>
+ * Input:
+ * <pre>
+ * FROM ...
+ * WHERE ...
+ * SELECT * EXCLUDE a, b.f
+ * ORDER BY c
+ * </pre>
+ * Output:
+ * <pre>
+ * FROM ( FROM ...
+ * WHERE ...
+ * SELECT *
+ * ORDER BY c ) TMP_1
+ * SELECT VALUE OBJECT_REMOVE_FIELDS(TMP_1, [ "a", [ "b", "f" ]])
+ * </pre>
+ * <p>
+ * There exists a special case with a single {@link FromTerm} node (with no other local bindings) and a SELECT * clause,
+ * where we qualify our field exclusion list with the {@link FromTerm} variable if we cannot anchor on the dataset
+ * variable. For example:
+ * <pre>
+ * FROM MyDataset D
+ * SELECT * EXCLUDE a, b.c, D.d
+ * </pre>
+ * Is conceptually processed as:
+ * <pre>
+ * FROM MyDataset D
+ * SELECT * EXCLUDE D.a, D.b.c, D.d
+ * </pre>
+ * For all other cases, our EXCLUDE will work solely with what our SELECT returns.
+ */
+public class SelectExcludeRewriteSugarVisitor extends AbstractSqlppExpressionScopingVisitor {
+ public SelectExcludeRewriteSugarVisitor(LangRewritingContext langRewritingContext) {
+ super(langRewritingContext);
+ }
+
+ @Override
+ public Expression visit(SelectBlock selectBlock, ILangExpression arg) throws CompilationException {
+ super.visit(selectBlock, arg);
+
+ // Proceed if we have field-exclusions.
+ SelectClause selectClause = selectBlock.getSelectClause();
+ if (selectClause.getFieldExclusions().isEmpty()) {
+ return null;
+ }
+ SelectExpression selectExpression = (SelectExpression) arg;
+
+ // If we have a single dataset in a FROM-CLAUSE (with no other variables in our local scope / grouping)...
+ if (selectBlock.hasFromClause() && selectBlock.getFromClause().getFromTerms().size() == 1) {
+ FromTerm fromTerm = selectBlock.getFromClause().getFromTerms().get(0);
+ if (!selectBlock.hasGroupbyClause() && !fromTerm.hasCorrelateClauses() && selectBlock.getLetWhereList()
+ .stream().noneMatch(c -> c.getClauseType() == Clause.ClauseType.LET_CLAUSE)) {
+ // ...and we have a 'SELECT *'...
+ SelectRegular selectRegular = selectClause.getSelectRegular();
+ if (selectClause.selectRegular() && selectRegular.getProjections().size() == 1
+ && selectRegular.getProjections().get(0).getKind() == Projection.Kind.STAR) {
+ // ...then qualify our field exclusions with our FROM-CLAUSE variable.
+ String fromTermName = fromTerm.getLeftVariable().getVar().getValue();
+ String qualifier = SqlppVariableUtil.toUserDefinedName(fromTermName);
+ selectClause.getFieldExclusions().stream().filter(e -> {
+ // Do not needlessly qualify names that are already bound to variables in our scope.
+ // Note: We use our local scope to include the single-dataset variable AND our outer scope.
+ // We already know that there are no other variables in our local scope.
+ Iterator<Pair<Identifier, Set<? extends Scope.SymbolAnnotation>>> liveSymbolIterator =
+ scopeChecker.getCurrentScope().liveSymbols(null);
+ while (liveSymbolIterator.hasNext()) {
+ Pair<Identifier, Set<? extends Scope.SymbolAnnotation>> symbol = liveSymbolIterator.next();
+ String symbolName = SqlppVariableUtil.toUserDefinedName(symbol.first.getValue());
+ if (symbolName.equals(e.get(0))) {
+ return false;
+ }
+ }
+ return true;
+ }).forEach(e -> e.add(0, qualifier));
+ }
+ }
+ }
+
+ // Find our parent SET-OP-INPUT.
+ SetOperationInput setOperationInput = null;
+ SelectSetOperation selectSetOperation = selectExpression.getSelectSetOperation();
+ if (selectBlock.equals(selectSetOperation.getLeftInput().getSelectBlock())) {
+ setOperationInput = selectSetOperation.getLeftInput();
+ } else {
+ for (SetOperationRight rightInput : selectSetOperation.getRightInputs()) {
+ SetOperationInput setOperationRightInput = rightInput.getSetOperationRightInput();
+ if (selectBlock.equals(setOperationRightInput.getSelectBlock())) {
+ setOperationInput = setOperationRightInput;
+ break;
+ }
+ }
+ }
+ if (setOperationInput == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, selectBlock.getSourceLocation(),
+ "Parent SET-OP-INPUT not found while rewriting SELECT-EXCLUDE!");
+ }
+
+ // Nest our original SELECT-BLOCK.
+ SourceLocation sourceLocation = selectBlock.getSourceLocation();
+ SetOperationInput innerSetOpInput = new SetOperationInput(selectBlock, null);
+ SelectSetOperation innerSelectSetOp = new SelectSetOperation(innerSetOpInput, null);
+ innerSelectSetOp.setSourceLocation(sourceLocation);
+ SelectExpression innerSelectExpr;
+ if (!selectSetOperation.hasRightInputs()) {
+ // We need to attach our LET / ORDER BY / LIMIT to our inner SELECT-EXPR.
+ SelectExpression selectExprCopy = (SelectExpression) SqlppRewriteUtil.deepCopy(selectExpression);
+ innerSelectExpr = new SelectExpression(selectExprCopy.getLetList(), innerSelectSetOp,
+ selectExprCopy.getOrderbyClause(), selectExprCopy.getLimitClause(), true);
+ selectExpression.getLetList().clear();
+ selectExpression.setOrderbyClause(null);
+ selectExpression.setLimitClause(null);
+ } else {
+ innerSelectExpr = new SelectExpression(null, innerSelectSetOp, null, null, true);
+ }
+ innerSelectExpr.setSourceLocation(sourceLocation);
+
+ // Build a new SELECT-BLOCK.
+ VarIdentifier fromTermVariable = context.newVariable();
+ VariableExpr fromTermVariableExpr = new VariableExpr(fromTermVariable);
+ SelectClause innerSelectClause = buildSelectClause(selectClause, fromTermVariable);
+ innerSelectClause.setSourceLocation(sourceLocation);
+ FromTerm innerFromTerm = new FromTerm(innerSelectExpr, fromTermVariableExpr, null, null);
+ innerFromTerm.setSourceLocation(sourceLocation);
+ FromClause innerFromClause = new FromClause(List.of(innerFromTerm));
+ innerFromClause.setSourceLocation(sourceLocation);
+ SelectBlock innerSelectBlock = new SelectBlock(innerSelectClause, innerFromClause, null, null, null);
+ setOperationInput.setSelectBlock(innerSelectBlock);
+ return null;
+ }
+
+ private SelectClause buildSelectClause(SelectClause originalSelectClause, VarIdentifier iterationVariable) {
+ // Convert our list of identifiers into a list of literals representing field names.
+ ListConstructor listConstructor = new ListConstructor();
+ listConstructor.setType(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR);
+ listConstructor.setExprList(new ArrayList<>());
+ for (List<String> nestedField : originalSelectClause.getFieldExclusions()) {
+ if (nestedField.size() == 1) {
+ // For non-nested fields, we do not wrap our name in a list.
+ listConstructor.getExprList().add(new LiteralExpr(new StringLiteral(nestedField.get(0))));
+ } else {
+ // Otherwise, build a list to insert into our list.
+ ListConstructor nestedFieldList = new ListConstructor();
+ nestedFieldList.setType(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR);
+ nestedFieldList.setExprList(nestedField.stream().map(f -> new LiteralExpr(new StringLiteral(f)))
+ .collect(Collectors.toList()));
+ listConstructor.getExprList().add(nestedFieldList);
+ }
+ }
+ List<Expression> objectRemoveFieldsArguments = new ArrayList<>();
+ objectRemoveFieldsArguments.add(new VariableExpr(iterationVariable));
+ objectRemoveFieldsArguments.add(listConstructor);
+ originalSelectClause.getFieldExclusions().clear();
+
+ // Remove the DISTINCT from our original SELECT-CLAUSE, if it exists.
+ boolean isDistinct = originalSelectClause.distinct();
+ if (isDistinct) {
+ originalSelectClause.setDistinct(false);
+ }
+
+ // Create the call to OBJECT_REMOVE_FIELDS.
+ FunctionSignature functionSignature = new FunctionSignature(BuiltinFunctions.REMOVE_FIELDS);
+ CallExpr callExpr = new CallExpr(functionSignature, objectRemoveFieldsArguments);
+ SelectElement selectElement = new SelectElement(callExpr);
+ SelectClause selectClause = new SelectClause(selectElement, null, isDistinct);
+ selectClause.setSourceLocation(originalSelectClause.getSourceLocation());
+ return selectClause;
+ }
+}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java
index d337de9..c4fb951 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java
@@ -124,7 +124,8 @@
return expr.accept(visitor, selectBlock);
}
- private List<Pair<Expression, Identifier>> createGroupFieldList(SelectBlock selectBlock) {
+ private List<Pair<Expression, Identifier>> createGroupFieldList(SelectBlock selectBlock)
+ throws CompilationException {
List<Pair<Expression, Identifier>> groupFieldList = new ArrayList<>();
addToFieldList(groupFieldList, SqlppVariableUtil.getBindingVariables(selectBlock.getFromClause()));
addToFieldList(groupFieldList, SqlppVariableUtil.getLetBindingVariables(selectBlock.getLetWhereList()));
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java
index bf3e227..aa548519 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java
@@ -139,7 +139,8 @@
winExpr.setExprList(newExprList);
}
- private List<Pair<Expression, Identifier>> createWindowFieldList(SelectBlock selectBlock) {
+ private List<Pair<Expression, Identifier>> createWindowFieldList(SelectBlock selectBlock)
+ throws CompilationException {
List<Pair<Expression, Identifier>> fieldList = new ArrayList<>();
if (selectBlock != null) {
addToFieldList(fieldList, SqlppVariableUtil.getBindingVariables(selectBlock.getFromClause()));
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
index 9d50160..0a8e528 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
@@ -29,17 +29,11 @@
import org.apache.asterix.lang.common.base.Clause;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
-import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
-import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
-import org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.struct.Identifier;
-import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
-import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
-import org.apache.asterix.lang.sqlpp.clause.FromClause;
-import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.visitor.BindingVariableVisitor;
import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -131,61 +125,14 @@
return freeVars;
}
- public static List<VariableExpr> getBindingVariables(FromClause fromClause) {
- if (fromClause == null) {
+ public static List<VariableExpr> getBindingVariables(ILangExpression langExpr) throws CompilationException {
+ if (langExpr == null) {
return Collections.emptyList();
}
- List<VariableExpr> bindingVars = new ArrayList<>();
- for (FromTerm fromTerm : fromClause.getFromTerms()) {
- bindingVars.addAll(getBindingVariables(fromTerm));
- }
- return bindingVars;
- }
- public static List<VariableExpr> getBindingVariables(FromTerm fromTerm) {
+ final BindingVariableVisitor visitor = new BindingVariableVisitor();
List<VariableExpr> bindingVars = new ArrayList<>();
- if (fromTerm == null) {
- return bindingVars;
- }
- bindingVars.add(fromTerm.getLeftVariable());
- if (fromTerm.hasPositionalVariable()) {
- bindingVars.add(fromTerm.getPositionalVariable());
- }
- for (AbstractBinaryCorrelateClause correlateClause : fromTerm.getCorrelateClauses()) {
- bindingVars.add(correlateClause.getRightVariable());
- if (correlateClause.hasPositionalVariable()) {
- bindingVars.add(correlateClause.getPositionalVariable());
- }
- }
- return bindingVars;
- }
-
- public static List<VariableExpr> getBindingVariables(GroupbyClause gbyClause) {
- List<VariableExpr> bindingVars = new ArrayList<>();
- if (gbyClause == null) {
- return bindingVars;
- }
- Set<VariableExpr> gbyKeyVars = new HashSet<>();
- for (List<GbyVariableExpressionPair> gbyPairList : gbyClause.getGbyPairList()) {
- for (GbyVariableExpressionPair gbyKey : gbyPairList) {
- VariableExpr var = gbyKey.getVar();
- if (var != null && gbyKeyVars.add(var)) {
- bindingVars.add(var);
- }
- }
- }
- if (gbyClause.hasDecorList()) {
- for (GbyVariableExpressionPair gbyKey : gbyClause.getDecorPairList()) {
- VariableExpr var = gbyKey.getVar();
- if (var != null) {
- bindingVars.add(var);
- }
- }
- }
- if (gbyClause.hasWithMap()) {
- bindingVars.addAll(gbyClause.getWithVarMap().values());
- }
- bindingVars.add(gbyClause.getGroupVar());
+ langExpr.accept(visitor, bindingVars);
return bindingVars;
}
@@ -203,15 +150,6 @@
return bindingVars;
}
- public static List<VariableExpr> getBindingVariables(QuantifiedExpression qe) {
- List<QuantifiedPair> quantifiedList = qe.getQuantifiedList();
- List<VariableExpr> bindingVars = new ArrayList<>(quantifiedList.size());
- for (QuantifiedPair qp : quantifiedList) {
- bindingVars.add(qp.getVarExpr());
- }
- return bindingVars;
- }
-
public static void addToFieldVariableList(VariableExpr varExpr, List<Pair<Expression, Identifier>> outFieldList) {
VarIdentifier var = varExpr.getVar();
VariableExpr newVarExpr = new VariableExpr(var);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/BindingVariableVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/BindingVariableVisitor.java
new file mode 100644
index 0000000..1b7dc24
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/BindingVariableVisitor.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.visitor;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.struct.QuantifiedPair;
+import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppAstVisitor;
+
+public class BindingVariableVisitor extends AbstractSqlppAstVisitor<Void, Collection<VariableExpr>> {
+ @Override
+ public Void visit(FromClause fromClause, Collection<VariableExpr> bindingVars) throws CompilationException {
+ for (FromTerm fromTerm : fromClause.getFromTerms()) {
+ fromTerm.accept(this, bindingVars);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(FromTerm fromTerm, Collection<VariableExpr> bindingVars) throws CompilationException {
+ bindingVars.add(fromTerm.getLeftVariable());
+ if (fromTerm.hasPositionalVariable()) {
+ bindingVars.add(fromTerm.getPositionalVariable());
+ }
+ for (AbstractBinaryCorrelateClause correlateClause : fromTerm.getCorrelateClauses()) {
+ bindingVars.add(correlateClause.getRightVariable());
+ if (correlateClause.hasPositionalVariable()) {
+ bindingVars.add(correlateClause.getPositionalVariable());
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(GroupbyClause groupbyClause, Collection<VariableExpr> bindingVars) throws CompilationException {
+ Set<VariableExpr> gbyKeyVars = new HashSet<>();
+ for (List<GbyVariableExpressionPair> gbyPairList : groupbyClause.getGbyPairList()) {
+ for (GbyVariableExpressionPair gbyKey : gbyPairList) {
+ VariableExpr var = gbyKey.getVar();
+ if (var != null && gbyKeyVars.add(var)) {
+ bindingVars.add(var);
+ }
+ }
+ }
+ if (groupbyClause.hasDecorList()) {
+ for (GbyVariableExpressionPair gbyKey : groupbyClause.getDecorPairList()) {
+ VariableExpr var = gbyKey.getVar();
+ if (var != null) {
+ bindingVars.add(var);
+ }
+ }
+ }
+ if (groupbyClause.hasWithMap()) {
+ bindingVars.addAll(groupbyClause.getWithVarMap().values());
+ }
+ bindingVars.add(groupbyClause.getGroupVar());
+ return null;
+ }
+
+ @Override
+ public Void visit(QuantifiedExpression qe, Collection<VariableExpr> bindingVars) throws CompilationException {
+ List<QuantifiedPair> quantifiedList = qe.getQuantifiedList();
+ for (QuantifiedPair qp : quantifiedList) {
+ bindingVars.add(qp.getVarExpr());
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(IVisitorExtension ve, Collection<VariableExpr> arg) throws CompilationException {
+ return ve.bindingVariableDispatch(this, arg);
+ }
+}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
index e4ccef5..f700d21 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
@@ -21,6 +21,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -160,6 +161,11 @@
}
@Override
+ public Boolean visit(IVisitorExtension ve, VariableExpr arg) throws CompilationException {
+ return ve.checkDatasetOnlyDispatch(this, arg);
+ }
+
+ @Override
public Boolean visit(IfExpr ifexpr, VariableExpr arg) throws CompilationException {
return false;
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java
index d6ee1fc..d16322e 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java
@@ -22,6 +22,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppContainsExpressionVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -68,4 +69,9 @@
}
return super.visit(callExpr, arg);
}
+
+ @Override
+ public Boolean visit(IVisitorExtension ve, Void arg) throws CompilationException {
+ return ve.checkNonFunctionalDispatch(this);
+ }
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSql92AggregateVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSql92AggregateVisitor.java
index 3192541..5e6a84a 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSql92AggregateVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSql92AggregateVisitor.java
@@ -24,6 +24,7 @@
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -157,6 +158,11 @@
}
@Override
+ public Boolean visit(IVisitorExtension ve, ILangExpression arg) throws CompilationException {
+ return ve.check92AggregateDispatch(this, arg);
+ }
+
+ @Override
public Boolean visit(IfExpr ifexpr, ILangExpression parentSelectBlock) throws CompilationException {
if (ifexpr.getCondExpr().accept(this, parentSelectBlock)) {
return true;
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
index a903279..399e463 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
@@ -24,6 +24,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -231,6 +232,11 @@
}
@Override
+ public Boolean visit(IVisitorExtension ve, ILangExpression arg) throws CompilationException {
+ return ve.checkSubqueryDispatch(this, arg);
+ }
+
+ @Override
public Boolean visit(IfExpr ifexpr, ILangExpression arg) throws CompilationException {
return visit(ifexpr.getCondExpr(), arg) || visit(ifexpr.getThenExpr(), arg) || visit(ifexpr.getElseExpr(), arg);
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java
index d74d047..1d43d0b 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java
@@ -28,6 +28,7 @@
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -193,7 +194,14 @@
if (selectClause.selectRegular()) {
selectRegular = (SelectRegular) selectClause.getSelectRegular().accept(this, arg);
}
- SelectClause copy = new SelectClause(selectElement, selectRegular, selectClause.distinct());
+ List<List<String>> fieldExclusions = new ArrayList<>();
+ if (!selectClause.getFieldExclusions().isEmpty()) {
+ for (List<String> fieldExclusion : selectClause.getFieldExclusions()) {
+ List<String> fieldExclusionCopy = new ArrayList<>(fieldExclusion);
+ fieldExclusions.add(fieldExclusionCopy);
+ }
+ }
+ SelectClause copy = new SelectClause(selectElement, selectRegular, fieldExclusions, selectClause.distinct());
copy.setSourceLocation(selectClause.getSourceLocation());
return copy;
}
@@ -507,6 +515,11 @@
}
@Override
+ public ILangExpression visit(IVisitorExtension ve, Void arg) throws CompilationException {
+ return ve.deepCopyDispatch(this);
+ }
+
+ @Override
public ILangExpression visit(CaseExpression caseExpr, Void arg) throws CompilationException {
Expression conditionExpr = (Expression) caseExpr.getConditionExpr().accept(this, arg);
List<Expression> whenExprList = copyExprList(caseExpr.getWhenExprs(), arg);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
index 9115b1c..8ab87e7 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
@@ -27,6 +27,7 @@
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.Clause.ClauseType;
import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -468,6 +469,12 @@
}
@Override
+ public Void visit(IVisitorExtension ve, Collection<VariableExpr> arg) throws CompilationException {
+ ve.freeVariableDispatch(this, arg);
+ return null;
+ }
+
+ @Override
public Void visit(CaseExpression caseExpr, Collection<VariableExpr> freeVars) throws CompilationException {
caseExpr.getConditionExpr().accept(this, freeVars);
visit(caseExpr.getWhenExprs(), freeVars);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java
index c803d83..cab6e87 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java
@@ -22,12 +22,14 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DatasetFullyQualifiedName;
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.base.Literal;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
@@ -178,6 +180,11 @@
public Void visit(SelectClause selectClause, Integer step) throws CompilationException {
if (selectClause.selectRegular()) {
selectClause.getSelectRegular().accept(this, step);
+ if (!selectClause.getFieldExclusions().isEmpty()) {
+ out.print(skip(step) + "EXCLUDE ");
+ out.println(selectClause.getFieldExclusions().stream().map(e -> String.join(".", e))
+ .collect(Collectors.joining(",")));
+ }
}
if (selectClause.selectElement()) {
selectClause.getSelectElement().accept(this, step);
@@ -373,6 +380,12 @@
}
@Override
+ public Void visit(IVisitorExtension ve, Integer arg) throws CompilationException {
+ // Language extensions should create a child of this class.
+ return null;
+ }
+
+ @Override
public Void visit(WindowExpression winExpr, Integer step) throws CompilationException {
out.print(skip(step) + "WINDOW ");
printFunctionSignature(out, winExpr.getFunctionSignature(), winExpr.getFunctionSignature().getArity());
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java
index ce0d0a1..bff57e2 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java
@@ -269,7 +269,15 @@
} else {
Pair<ILangExpression, VariableSubstitutionEnvironment> newSelectRegular =
selectClause.getSelectRegular().accept(this, env);
- SelectClause newSelectClause = new SelectClause(null, (SelectRegular) newSelectRegular.first, distinct);
+ List<List<String>> fieldExclusions = new ArrayList<>();
+ if (!selectClause.getFieldExclusions().isEmpty()) {
+ for (List<String> fieldExclusion : selectClause.getFieldExclusions()) {
+ List<String> fieldExclusionCopy = new ArrayList<>(fieldExclusion);
+ fieldExclusions.add(fieldExclusionCopy);
+ }
+ }
+ SelectClause newSelectClause =
+ new SelectClause(null, (SelectRegular) newSelectRegular.first, fieldExclusions, distinct);
newSelectClause.setSourceLocation(selectClause.getSourceLocation());
return new Pair<>(newSelectClause, newSelectRegular.second);
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java
index 975379d..4fa6bdc 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java
@@ -20,10 +20,12 @@
import java.io.PrintWriter;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
@@ -171,6 +173,10 @@
public Void visit(SelectClause selectClause, Integer step) throws CompilationException {
if (selectClause.selectRegular()) {
selectClause.getSelectRegular().accept(this, step);
+ if (!selectClause.getFieldExclusions().isEmpty()) {
+ out.println("exclude " + selectClause.getFieldExclusions().stream().map(e -> String.join(".", e))
+ .collect(Collectors.joining(COMMA)));
+ }
}
if (selectClause.selectElement()) {
selectClause.getSelectElement().accept(this, step);
@@ -284,6 +290,12 @@
}
@Override
+ public Void visit(IVisitorExtension ve, Integer arg) throws CompilationException {
+ // Language extensions should create a child of this class.
+ return null;
+ }
+
+ @Override
public Void visit(InsertStatement insert, Integer step) throws CompilationException {
out.print(skip(step) + "insert into " + datasetSymbol
+ generateFullName(insert.getDataverseName(), insert.getDatasetName()) + "\n");
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppAstVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppAstVisitor.java
index e9c0e4c..7973841 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppAstVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppAstVisitor.java
@@ -19,6 +19,7 @@
package org.apache.asterix.lang.sqlpp.visitor.base;
import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.expression.ListSliceExpression;
import org.apache.asterix.lang.common.visitor.base.AbstractAstVisitor;
import org.apache.asterix.lang.sqlpp.clause.FromClause;
@@ -121,4 +122,9 @@
public R visit(ListSliceExpression expression, T arg) throws CompilationException {
return null;
}
+
+ @Override
+ public R visit(IVisitorExtension ve, T arg) throws CompilationException {
+ return null;
+ }
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
index 7539046..765c2be 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
@@ -32,6 +32,7 @@
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -413,6 +414,11 @@
return winExpr;
}
+ @Override
+ public Expression visit(IVisitorExtension ve, ILangExpression arg) throws CompilationException {
+ return ve.variableScopeDispatch(this, arg, scopeChecker);
+ }
+
// Adds a new encountered alias identifier into a scope
private void addNewVarSymbolToScope(Scope scope, VarIdentifier var, SourceLocation sourceLoc,
SqlppVariableAnnotation... varAnnotations) throws CompilationException {
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
index ef8b43c..6bbb740 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
@@ -25,6 +25,7 @@
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.IVisitorExtension;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
@@ -384,6 +385,11 @@
}
@Override
+ public Expression visit(IVisitorExtension ve, ILangExpression arg) throws CompilationException {
+ return ve.simpleExpressionDispatch(this, arg);
+ }
+
+ @Override
public Expression visit(CaseExpression caseExpr, ILangExpression arg) throws CompilationException {
caseExpr.setConditionExpr(visit(caseExpr.getConditionExpr(), arg));
caseExpr.setWhenExprs(visit(caseExpr.getWhenExprs(), arg));
@@ -403,7 +409,7 @@
return null;
}
- protected Expression visit(Expression expr, ILangExpression arg) throws CompilationException {
+ public Expression visit(Expression expr, ILangExpression arg) throws CompilationException {
return postVisit(preVisit(expr).accept(this, arg));
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index c7ded7b..1c4c977 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -171,7 +171,6 @@
import org.apache.asterix.lang.common.statement.UpsertStatement;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.statement.ViewDropStatement;
-import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
@@ -931,7 +930,6 @@
| stmt = CreateStatement()
| stmt = LoadStatement()
| stmt = DropStatement()
- | stmt = WriteStatement()
| stmt = SetStatement()
| stmt = InsertStatement()
| stmt = DeleteStatement()
@@ -2680,24 +2678,6 @@
}
}
-Statement WriteStatement() throws ParseException:
-{
- Token startToken = null;
- String nodeName = null;
- String fileName = null;
- Query query;
- String writerClass = null;
- Pair<Identifier,Identifier> nameComponents = null;
-}
-{
- <WRITE> { startToken = token; } <OUTPUT> <TO> nodeName = Identifier() <COLON> fileName = ConstantString()
- ( <USING> writerClass = ConstantString() )?
- {
- WriteStatement stmt = new WriteStatement(new Identifier(nodeName), fileName, writerClass);
- return addSourceLocation(stmt, startToken);
- }
-}
-
LoadStatement LoadStatement() throws ParseException:
{
Token startToken = null;
@@ -4879,12 +4859,33 @@
Token startToken = null;
SelectRegular selectRegular = null;
SelectElement selectElement = null;
+ List<List<String>> fieldExclusions = new ArrayList<List<String>>();
+ List<String> nestedField = new ArrayList<String>();
+ String identifier;
boolean distinct = false;
}
{
<SELECT> { startToken = token; } (<ALL>|<DISTINCT> { distinct = true; } )?
(
- selectRegular = SelectRegular()
+ (
+ selectRegular = SelectRegular()
+ ( LOOKAHEAD({laIdentifier(EXCLUDE)}) <IDENTIFIER>
+ identifier = Identifier() { nestedField.add(identifier); }
+ ( <DOT> identifier = Identifier() { nestedField.add(identifier); } )*
+ {
+ fieldExclusions.add(nestedField);
+ nestedField = new ArrayList<String>();
+ }
+ ( LOOKAHEAD(1) // Force <COMMA> to be recognized for a nested field in our EXCLUDE list.
+ <COMMA> identifier = Identifier() { nestedField.add(identifier); }
+ ( <DOT> identifier = Identifier() { nestedField.add(identifier); } )*
+ {
+ fieldExclusions.add(nestedField);
+ nestedField = new ArrayList<String>();
+ }
+ )*
+ )?
+ )
|
selectElement = SelectElement()
)?
@@ -4898,7 +4899,7 @@
selectRegular = new SelectRegular(projections);
selectRegular.setSourceLocation(sourceLoc);
}
- SelectClause selectClause = new SelectClause(selectElement, selectRegular, distinct);
+ SelectClause selectClause = new SelectClause(selectElement, selectRegular, fieldExclusions, distinct);
selectClause.setSourceLocation(sourceLoc);
return selectClause;
}
@@ -4956,7 +4957,12 @@
(
<MUL> { kind = Projection.Kind.STAR; startSrcLoc = getSourceLocation(token); }
| LOOKAHEAD(3) expr = VariableRef() <DOT> <MUL> { kind = Projection.Kind.VAR_STAR; }
- | expr = Expression() ((<AS>)? name = Identifier())?
+ | expr = Expression()
+ ( // EXCLUDE is a soft-keyword-- we want to avoid mistaking EXCLUDE as an identifier here.
+ LOOKAHEAD({ getToken(1).kind == AS || getToken(1).kind == QUOTED_STRING
+ || (getToken(1).kind == IDENTIFIER && !laIdentifier(1, EXCLUDE)) })
+ (<AS>)? name = Identifier()
+ )?
{
kind = Projection.Kind.NAMED_EXPR;
if (name == null) {
diff --git a/asterixdb/asterix-license/pom.xml b/asterixdb/asterix-license/pom.xml
index 3a6f1c3..fcdd548 100644
--- a/asterixdb/asterix-license/pom.xml
+++ b/asterixdb/asterix-license/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
index cd00397..0d72872 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-maven-plugins</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-grammar-extension-maven-plugin</artifactId>
<packaging>maven-plugin</packaging>
diff --git a/asterixdb/asterix-maven-plugins/asterix-test-datagenerator-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/asterix-test-datagenerator-maven-plugin/pom.xml
index 88aaf2b..a322c47 100644
--- a/asterixdb/asterix-maven-plugins/asterix-test-datagenerator-maven-plugin/pom.xml
+++ b/asterixdb/asterix-maven-plugins/asterix-test-datagenerator-maven-plugin/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>asterix-maven-plugins</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/asterixdb/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml
index dccb9d7..5bba5e8 100644
--- a/asterixdb/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml
+++ b/asterixdb/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>asterix-maven-plugins</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<packaging>maven-plugin</packaging>
diff --git a/asterixdb/asterix-maven-plugins/pom.xml b/asterixdb/asterix-maven-plugins/pom.xml
index fd15ac5..ac20f8c 100644
--- a/asterixdb/asterix-maven-plugins/pom.xml
+++ b/asterixdb/asterix-maven-plugins/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/asterixdb/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml
index ab1b13a..98f3511 100644
--- a/asterixdb/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml
+++ b/asterixdb/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>asterix-maven-plugins</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<packaging>maven-plugin</packaging>
diff --git a/asterixdb/asterix-metadata/pom.xml b/asterixdb/asterix-metadata/pom.xml
index 61507d1..e0b5387 100644
--- a/asterixdb/asterix-metadata/pom.xml
+++ b/asterixdb/asterix-metadata/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-metadata</artifactId>
<properties>
@@ -180,5 +180,10 @@
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-ipc</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-column</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
index f00090a..f1156f7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
@@ -196,6 +196,10 @@
"BlockLevelStorageCompression";
public static final String DATASET_ARECORD_DATASET_COMPRESSION_SCHEME_FIELD_NAME = "DatasetCompressionScheme";
public static final String DATASET_ARECORD_REBALANCE_FIELD_NAME = "rebalanceCount";
+ public static final String DATASET_ARECORD_DATASET_FORMAT_FIELD_NAME = "DatasetFormat";
+ public static final String DATASET_ARECORD_DATASET_FORMAT_FORMAT_FIELD_NAME = "Format";
+ public static final String DATASET_ARECORD_DATASET_MAX_TUPLE_COUNT_FIELD_NAME = "MaxTupleCount";
+ public static final String DATASET_ARECORD_DATASET_FREE_SPACE_TOLERANCE_FIELD_NAME = "FreeSpaceTolerance";
public static final ARecordType DATASET_RECORDTYPE = createRecordType(
// RecordTypeName
RECORD_NAME_DATASET,
@@ -574,7 +578,7 @@
return recordType;
}
- public static final ARecordType createPropertiesRecordType() {
+ public static ARecordType createPropertiesRecordType() {
return createRecordType(
// RecordTypeName
null,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/DatasetFormatInfo.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/DatasetFormatInfo.java
new file mode 100644
index 0000000..38951a4
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/DatasetFormatInfo.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.dataset;
+
+import java.io.Serializable;
+
+import org.apache.asterix.common.config.DatasetConfig.DatasetFormat;
+
+public class DatasetFormatInfo implements Serializable {
+ private static final long serialVersionUID = 7656132322813253435L;
+ public static final DatasetFormatInfo DEFAULT = new DatasetFormatInfo();
+ private final DatasetFormat format;
+ private final int maxTupleCount;
+ private final float freeSpaceTolerance;
+
+ private DatasetFormatInfo() {
+ this(DatasetFormat.ROW, -1, 0.0f);
+ }
+
+ public DatasetFormatInfo(DatasetFormat format, int maxTupleCount, float freeSpaceTolerance) {
+ this.format = format;
+ this.maxTupleCount = maxTupleCount;
+ this.freeSpaceTolerance = freeSpaceTolerance;
+ }
+
+ public DatasetFormat getFormat() {
+ return format;
+ }
+
+ public int getMaxTupleCount() {
+ return maxTupleCount;
+ }
+
+ public float getFreeSpaceTolerance() {
+ return freeSpaceTolerance;
+ }
+
+ @Override
+ public String toString() {
+ return "(format:" + format + ", maxTupleCount:" + maxTupleCount + ')';
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
index d119067..cafa001 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
@@ -21,6 +21,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.column.ColumnManagerFactory;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -31,8 +32,10 @@
import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.formats.nontagged.NullIntrospector;
import org.apache.asterix.metadata.api.IResourceFactoryProvider;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.BuiltinType;
@@ -45,6 +48,8 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManagerFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.dataflow.LSMColumnBTreeLocalResourceFactory;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeLocalResourceFactory;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyLocalResourceFactory;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResourceFactory;
@@ -115,13 +120,30 @@
boolean isSecondaryNoIncrementalMaintenance = index.getIndexType() == DatasetConfig.IndexType.SAMPLE;
- return new LSMBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits,
- filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory,
- pageWriteCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider,
- mergePolicyFactory, mergePolicyProperties, true, bloomFilterFields,
- bloomFilterFalsePositiveRate, index.isPrimaryIndex(), btreeFields, compDecompFactory,
- hasBloomFilter, typeTraitProvider.getTypeTrait(BuiltinType.ANULL), NullIntrospector.INSTANCE,
- isSecondaryNoIncrementalMaintenance);
+ DatasetFormatInfo datasetFormatInfo = dataset.getDatasetFormatInfo();
+ if (!index.isPrimaryIndex() || datasetFormatInfo.getFormat() == DatasetConfig.DatasetFormat.ROW) {
+ return new LSMBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits,
+ filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory,
+ pageWriteCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider,
+ mergePolicyFactory, mergePolicyProperties, true, bloomFilterFields,
+ bloomFilterFalsePositiveRate, index.isPrimaryIndex(), btreeFields, compDecompFactory,
+ hasBloomFilter, typeTraitProvider.getTypeTrait(BuiltinType.ANULL),
+ NullIntrospector.INSTANCE, isSecondaryNoIncrementalMaintenance);
+ } else {
+ //Column
+ List<Integer> keySourceIndicator =
+ ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
+ IColumnManagerFactory columnManagerFactory =
+ new ColumnManagerFactory(recordType, metaType, dataset.getPrimaryKeys(), keySourceIndicator,
+ mdProvider.getStorageProperties().getBufferCachePageSize(),
+ datasetFormatInfo.getMaxTupleCount(), datasetFormatInfo.getFreeSpaceTolerance());
+ return new LSMColumnBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories,
+ filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory,
+ pageWriteCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider,
+ mergePolicyFactory, mergePolicyProperties, bloomFilterFields, bloomFilterFalsePositiveRate,
+ btreeFields, compDecompFactory, typeTraitProvider.getTypeTrait(BuiltinType.ANULL),
+ NullIntrospector.INSTANCE, isSecondaryNoIncrementalMaintenance, columnManagerFactory);
+ }
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE,
dataset.getDatasetType().toString());
@@ -156,9 +178,7 @@
secondaryTypeTraits[i] = typeTraitProvider.getTypeTrait(keyType);
}
// Add serializers and comparators for primary index fields.
- for (int i = 0; i < numPrimaryKeys; i++) {
- secondaryTypeTraits[numSecondaryKeys + i] = primaryTypeTraits[i];
- }
+ System.arraycopy(primaryTypeTraits, 0, secondaryTypeTraits, numSecondaryKeys, numPrimaryKeys);
return secondaryTypeTraits;
}
@@ -193,9 +213,7 @@
secondaryCmpFactories[i] = cmpFactoryProvider.getBinaryComparatorFactory(keyType, true);
}
// Add serializers and comparators for primary index fields.
- for (int i = 0; i < numPrimaryKeys; i++) {
- secondaryCmpFactories[numSecondaryKeys + i] = primaryCmpFactories[i];
- }
+ System.arraycopy(primaryCmpFactories, 0, secondaryCmpFactories, numSecondaryKeys, numPrimaryKeys);
return secondaryCmpFactories;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
index 95b0906..4e6119f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
@@ -33,7 +33,7 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
@@ -122,6 +122,7 @@
return datasourceType;
}
+ @Override
public Map<String, Serializable> getProperties() {
return properties;
}
@@ -165,5 +166,6 @@
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException;
+ IProjectionFiltrationInfo<?> datasetProjectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 66ea5a7..dbcbce0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -37,17 +37,18 @@
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
@@ -55,6 +56,7 @@
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
public class DatasetDataSource extends DataSource {
@@ -119,15 +121,16 @@
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException {
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException {
+ String itemTypeName = dataset.getItemTypeName();
+ IAType itemType = MetadataManager.INSTANCE
+ .getDatatype(metadataProvider.getMetadataTxnContext(), dataset.getItemTypeDataverseName(), itemTypeName)
+ .getDatatype();
switch (dataset.getDatasetType()) {
case EXTERNAL:
DatasetDataSource externalDataSource = (DatasetDataSource) dataSource;
Dataset externalDataset = externalDataSource.getDataset();
- String itemTypeName = externalDataset.getItemTypeName();
- IAType itemType = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
-
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
int externalScanBufferSize = physicalOptimizationConfig.getExternalScanBufferSize();
@@ -145,25 +148,38 @@
Index primaryIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
dataverseName, datasetName, datasetName);
+ ARecordType datasetType = (ARecordType) itemType;
+ ARecordType metaItemType = null;
+ if (dataset.hasMetaPart()) {
+ metaItemType = (ARecordType) MetadataManager.INSTANCE
+ .getDatatype(metadataProvider.getMetadataTxnContext(),
+ dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName())
+ .getDatatype();
+ }
+ int numberOfPrimaryKeys = dataset.getPrimaryKeys().size();
+ ITupleProjectorFactory tupleProjectorFactory =
+ IndexUtil.createTupleProjectorFactory(dataset.getDatasetFormatInfo(), projectionInfo,
+ metaProjectionInfo, datasetType, metaItemType, numberOfPrimaryKeys);
+
int[] minFilterFieldIndexes = createFilterIndexes(minFilterVars, opSchema);
int[] maxFilterFieldIndexes = createFilterIndexes(maxFilterVars, opSchema);
return metadataProvider.buildBtreeRuntime(jobSpec, opSchema, typeEnv, context, true, false, null,
((DatasetDataSource) dataSource).getDataset(), primaryIndex.getIndexName(), null, null, true,
true, false, null, minFilterFieldIndexes, maxFilterFieldIndexes, tupleFilterFactory,
- outputLimit, false, false);
+ outputLimit, false, false, tupleProjectorFactory);
default:
throw new AlgebricksException("Unknown datasource type");
}
}
- private Map<String, String> addExternalProjectionInfo(IProjectionInfo<?> projectionInfo,
+ private Map<String, String> addExternalProjectionInfo(IProjectionFiltrationInfo<?> projectionInfo,
Map<String, String> properties) {
Map<String, String> propertiesCopy = properties;
if (projectionInfo != null) {
//properties could be cached and reused, so we make a copy per query
propertiesCopy = new HashMap<>(properties);
try {
- DataProjectionInfo externalProjectionInfo = (DataProjectionInfo) projectionInfo;
+ DataProjectionFiltrationInfo externalProjectionInfo = (DataProjectionFiltrationInfo) projectionInfo;
ExternalDataUtils.setExternalDataProjectionInfo(externalProjectionInfo, propertiesCopy);
} catch (IOException e) {
throw new IllegalStateException(e);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index dc7b19b..0d65083 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -42,7 +42,7 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
@@ -164,7 +164,8 @@
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException {
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException {
try {
if (tupleFilterFactory != null || outputLimit >= 0) {
throw CompilationException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 5c874d7..4dadf75 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -41,7 +41,7 @@
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
@@ -94,7 +94,8 @@
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException {
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException {
GenericAdapterFactory adapterFactory = new GenericAdapterFactory();
adapterFactory.setOutputType(RecordUtil.FULLY_OPEN_RECORD_TYPE);
IClusterStateManager csm = metadataProvider.getApplicationContext().getClusterStateManager();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
index 79a9d00..c7ccc53 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
@@ -40,7 +40,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
@@ -136,7 +136,8 @@
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException {
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException {
if (tupleFilterFactory != null || outputLimit >= 0) {
throw CompilationException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
"tuple filter and limit are not supported by LoadableDataSource");
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 53cf3d0..c55925c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -130,7 +130,7 @@
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
@@ -174,6 +174,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
public class MetadataProvider implements IMetadataProvider<DataSourceId, String> {
@@ -493,10 +494,11 @@
List<LogicalVariable> projectVariables, boolean projectPushed, List<LogicalVariable> minFilterVars,
List<LogicalVariable> maxFilterVars, ITupleFilterFactory tupleFilterFactory, long outputLimit,
IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec,
- Object implConfig, IProjectionInfo<?> projectionInfo) throws AlgebricksException {
+ Object implConfig, IProjectionFiltrationInfo<?> projectionInfo,
+ IProjectionFiltrationInfo<?> metaProjectionInfo) throws AlgebricksException {
return ((DataSource) dataSource).buildDatasourceScanRuntime(this, dataSource, scanVariables, projectVariables,
projectPushed, minFilterVars, maxFilterVars, tupleFilterFactory, outputLimit, opSchema, typeEnv,
- context, jobSpec, implConfig, projectionInfo);
+ context, jobSpec, implConfig, projectionInfo, metaProjectionInfo);
}
protected Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildLoadableDatasetScan(
@@ -548,7 +550,8 @@
int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
boolean propagateFilter, IMissingWriterFactory nonFilterWriterFactory, int[] minFilterFieldIndexes,
int[] maxFilterFieldIndexes, ITupleFilterFactory tupleFilterFactory, long outputLimit,
- boolean isIndexOnlyPlan, boolean isPrimaryIndexPointSearch) throws AlgebricksException {
+ boolean isIndexOnlyPlan, boolean isPrimaryIndexPointSearch, ITupleProjectorFactory tupleProjectorFactory)
+ throws AlgebricksException {
boolean isSecondary = true;
Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), dataset.getDatasetName());
@@ -613,12 +616,13 @@
? new LSMBTreeBatchPointSearchOperatorDescriptor(jobSpec, outputRecDesc, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive, indexHelperFactory, retainInput,
retainMissing, nonMatchWriterFactory, searchCallbackFactory, minFilterFieldIndexes,
- maxFilterFieldIndexes, tupleFilterFactory, outputLimit)
+ maxFilterFieldIndexes, tupleFilterFactory, outputLimit, tupleProjectorFactory)
: new BTreeSearchOperatorDescriptor(jobSpec, outputRecDesc, lowKeyFields, highKeyFields,
lowKeyInclusive, highKeyInclusive, indexHelperFactory, retainInput, retainMissing,
nonMatchWriterFactory, searchCallbackFactory, minFilterFieldIndexes, maxFilterFieldIndexes,
propagateFilter, nonFilterWriterFactory, tupleFilterFactory, outputLimit,
- proceedIndexOnlyPlan, failValueForIndexOnlyPlan, successValueForIndexOnlyPlan);
+ proceedIndexOnlyPlan, failValueForIndexOnlyPlan, successValueForIndexOnlyPlan,
+ tupleProjectorFactory);
} else {
btreeSearchOp = new ExternalBTreeSearchOperatorDescriptor(jobSpec, outputRecDesc, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive, indexHelperFactory, retainInput, retainMissing,
@@ -873,7 +877,7 @@
*
* @param dataset
* @return Number of elements that will be used to create a bloom filter per
- * dataset per partition
+ * dataset per partition
* @throws AlgebricksException
*/
public long getCardinalityPerPartitionHint(Dataset dataset) throws AlgebricksException {
@@ -1502,12 +1506,8 @@
List<LogicalVariable> prevAdditionalFilteringKeys) throws AlgebricksException {
// Check the index is length-partitioned or not.
boolean isPartitioned;
- if (indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
- isPartitioned = true;
- } else {
- isPartitioned = false;
- }
+ isPartitioned = indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
// Sanity checks.
if (primaryKeys.size() > 1) {
@@ -1623,12 +1623,8 @@
}
boolean isPartitioned;
- if (indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
- isPartitioned = true;
- } else {
- isPartitioned = false;
- }
+ isPartitioned = indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
// Number of Keys that needs to be propagated
int numKeys = inputSchema.getSize();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/SampleDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/SampleDataSource.java
index 885723c..a2a0f19 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/SampleDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/SampleDataSource.java
@@ -29,13 +29,14 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
public class SampleDataSource extends DataSource {
@@ -59,10 +60,11 @@
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException {
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException {
return metadataProvider.buildBtreeRuntime(jobSpec, opSchema, typeEnv, context, true, false, null, dataset,
sampleIndexName, null, null, true, true, false, null, null, null, tupleFilterFactory, outputLimit,
- false, false);
+ false, false, DefaultTupleProjectorFactory.INSTANCE);
}
@Override
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index e3935a4..0774928 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -55,6 +55,7 @@
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.api.IMetadataEntity;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.declared.ArrayBTreeResourceFactoryProvider;
import org.apache.asterix.metadata.declared.BTreeResourceFactoryProvider;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -157,6 +158,7 @@
private int pendingOp;
private final String compressionScheme;
private final DatasetFullyQualifiedName datasetFullyQualifiedName;
+ private final DatasetFormatInfo datasetFormatInfo;
public Dataset(DataverseName dataverseName, String datasetName, DataverseName recordTypeDataverseName,
String recordTypeName, String nodeGroupName, String compactionPolicy,
@@ -164,17 +166,17 @@
DatasetType datasetType, int datasetId, int pendingOp) {
this(dataverseName, datasetName, recordTypeDataverseName, recordTypeName, /*metaTypeDataverseName*/null,
/*metaTypeName*/null, nodeGroupName, compactionPolicy, compactionPolicyProperties, datasetDetails,
- hints, datasetType, datasetId, pendingOp, CompressionManager.NONE);
+ hints, datasetType, datasetId, pendingOp, CompressionManager.NONE, DatasetFormatInfo.DEFAULT);
}
public Dataset(DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName,
String itemTypeName, DataverseName metaItemTypeDataverseName, String metaItemTypeName, String nodeGroupName,
String compactionPolicy, Map<String, String> compactionPolicyProperties, IDatasetDetails datasetDetails,
- Map<String, String> hints, DatasetType datasetType, int datasetId, int pendingOp,
- String compressionScheme) {
+ Map<String, String> hints, DatasetType datasetType, int datasetId, int pendingOp, String compressionScheme,
+ DatasetFormatInfo datasetFormatInfo) {
this(dataverseName, datasetName, itemTypeDataverseName, itemTypeName, metaItemTypeDataverseName,
metaItemTypeName, nodeGroupName, compactionPolicy, compactionPolicyProperties, datasetDetails, hints,
- datasetType, datasetId, pendingOp, 0L, compressionScheme);
+ datasetType, datasetId, pendingOp, 0L, compressionScheme, datasetFormatInfo);
}
public Dataset(Dataset dataset) {
@@ -182,14 +184,14 @@
dataset.metaTypeDataverseName, dataset.metaTypeName, dataset.nodeGroupName,
dataset.compactionPolicyFactory, dataset.compactionPolicyProperties, dataset.datasetDetails,
dataset.hints, dataset.datasetType, dataset.datasetId, dataset.pendingOp, dataset.rebalanceCount,
- dataset.compressionScheme);
+ dataset.compressionScheme, dataset.datasetFormatInfo);
}
public Dataset(DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName,
String itemTypeName, DataverseName metaItemTypeDataverseName, String metaItemTypeName, String nodeGroupName,
String compactionPolicy, Map<String, String> compactionPolicyProperties, IDatasetDetails datasetDetails,
Map<String, String> hints, DatasetType datasetType, int datasetId, int pendingOp, long rebalanceCount,
- String compressionScheme) {
+ String compressionScheme, DatasetFormatInfo datasetFormatInfo) {
this.dataverseName = dataverseName;
this.datasetName = datasetName;
this.recordTypeName = itemTypeName;
@@ -207,6 +209,7 @@
this.rebalanceCount = rebalanceCount;
this.compressionScheme = compressionScheme;
datasetFullyQualifiedName = new DatasetFullyQualifiedName(dataverseName, datasetName);
+ this.datasetFormatInfo = datasetFormatInfo;
}
@Override
@@ -332,21 +335,14 @@
/**
* Drop this dataset
*
- * @param metadataProvider
- * metadata provider that can be used to get metadata info and runtimes
- * @param mdTxnCtx
- * the transaction context
- * @param jobsToExecute
- * a list of jobs to be executed as part of the drop operation
- * @param bActiveTxn
- * whether the metadata transaction is ongoing
- * @param progress
- * a mutable progress state used for error handling during the drop operation
- * @param hcc
- * a client connection to hyracks master for job execution
+ * @param metadataProvider metadata provider that can be used to get metadata info and runtimes
+ * @param mdTxnCtx the transaction context
+ * @param jobsToExecute a list of jobs to be executed as part of the drop operation
+ * @param bActiveTxn whether the metadata transaction is ongoing
+ * @param progress a mutable progress state used for error handling during the drop operation
+ * @param hcc a client connection to hyracks master for job execution
* @param sourceLoc
- * @throws Exception
- * if an error occur during the drop process or if the dataset can't be dropped for any reason
+ * @throws Exception if an error occur during the drop process or if the dataset can't be dropped for any reason
*/
public void drop(MetadataProvider metadataProvider, MutableObject<MetadataTransactionContext> mdTxnCtx,
List<JobSpecification> jobsToExecute, MutableBoolean bActiveTxn, MutableObject<ProgressState> progress,
@@ -370,7 +366,8 @@
new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(),
getMetaItemTypeDataverseName(), getMetaItemTypeName(), getNodeGroupName(),
getCompactionPolicy(), getCompactionPolicyProperties(), getDatasetDetails(), getHints(),
- getDatasetType(), getDatasetId(), MetadataUtil.PENDING_DROP_OP, getCompressionScheme()));
+ getDatasetType(), getDatasetId(), MetadataUtil.PENDING_DROP_OP, getCompressionScheme(),
+ getDatasetFormatInfo()));
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
bActiveTxn.setValue(false);
@@ -450,22 +447,15 @@
/**
* Create the index dataflow helper factory for a particular index on the dataset
*
- * @param mdProvider
- * metadata provider to get metadata information, components, and runtimes
- * @param index
- * the index to get the dataflow helper factory for
- * @param recordType
- * the record type for the dataset
- * @param metaType
- * the meta type for the dataset
- * @param mergePolicyFactory
- * the merge policy factory of the dataset
- * @param mergePolicyProperties
- * the merge policy properties for the dataset
+ * @param mdProvider metadata provider to get metadata information, components, and runtimes
+ * @param index the index to get the dataflow helper factory for
+ * @param recordType the record type for the dataset
+ * @param metaType the meta type for the dataset
+ * @param mergePolicyFactory the merge policy factory of the dataset
+ * @param mergePolicyProperties the merge policy properties for the dataset
* @return indexDataflowHelperFactory
- * an instance of {@link org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory}
- * @throws AlgebricksException
- * if dataflow helper factory could not be created
+ * an instance of {@link org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory}
+ * @throws AlgebricksException if dataflow helper factory could not be created
*/
public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Index index, ARecordType recordType,
ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties)
@@ -511,13 +501,11 @@
/**
* Get the IO Operation callback factory for the index which belongs to this dataset
*
- * @param index
- * the index
+ * @param index the index
* @return ioOperationCallbackFactory
- * an instance of {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory}
- * to be used with IO operations
- * @throws AlgebricksException
- * if the factory could not be created for the index/dataset combination
+ * an instance of {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory}
+ * to be used with IO operations
+ * @throws AlgebricksException if the factory could not be created for the index/dataset combination
*/
@SuppressWarnings("squid:S1172")
public ILSMIOOperationCallbackFactory getIoOperationCallbackFactory(Index index) throws AlgebricksException {
@@ -531,8 +519,7 @@
/**
* get the IndexOperationTrackerFactory for a particular index on the dataset
*
- * @param index
- * the index
+ * @param index the index
* @return an instance of {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory}
*/
public ILSMOperationTrackerFactory getIndexOperationTrackerFactory(Index index) {
@@ -551,22 +538,14 @@
/**
* Get search callback factory for this dataset with the passed index and operation
*
- * @param storageComponentProvider
- * storage component provider
- * @param index
- * the index
- * @param op
- * the operation this search is part of
- * @param primaryKeyFields
- * the primary key fields indexes for locking purposes
- * @param primaryKeyFieldsInSecondaryIndex
- * the primary key fields indexes in the given secondary index (used for index-only plan)
- * @param proceedIndexOnlyPlan
- * the given plan is an index-only plan? (used for index-only plan)
- * @return
- * an instance of {@link org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory}
- * @throws AlgebricksException
- * if the callback factory could not be created
+ * @param storageComponentProvider storage component provider
+ * @param index the index
+ * @param op the operation this search is part of
+ * @param primaryKeyFields the primary key fields indexes for locking purposes
+ * @param primaryKeyFieldsInSecondaryIndex the primary key fields indexes in the given secondary index (used for index-only plan)
+ * @param proceedIndexOnlyPlan the given plan is an index-only plan? (used for index-only plan)
+ * @return an instance of {@link org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory}
+ * @throws AlgebricksException if the callback factory could not be created
*/
public ISearchOperationCallbackFactory getSearchCallbackFactory(IStorageComponentProvider storageComponentProvider,
Index index, IndexOperation op, int[] primaryKeyFields, int[] primaryKeyFieldsInSecondaryIndex,
@@ -599,18 +578,12 @@
/**
* Get search callback factory for this dataset with the passed index and operation
*
- * @param storageComponentProvider
- * storage component provider
- * @param index
- * the index
- * @param op
- * the operation this search is part of
- * @param primaryKeyFields
- * the primary key fields indexes for locking purposes
- * @return
- * an instance of {@link org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory}
- * @throws AlgebricksException
- * if the callback factory could not be created
+ * @param storageComponentProvider storage component provider
+ * @param index the index
+ * @param op the operation this search is part of
+ * @param primaryKeyFields the primary key fields indexes for locking purposes
+ * @return an instance of {@link org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory}
+ * @throws AlgebricksException if the callback factory could not be created
*/
public ISearchOperationCallbackFactory getSearchCallbackFactory(IStorageComponentProvider storageComponentProvider,
Index index, IndexOperation op, int[] primaryKeyFields) throws AlgebricksException {
@@ -620,16 +593,11 @@
/**
* Get the modification callback factory associated with this dataset, the passed index, and operation.
*
- * @param index
- * the index
- * @param op
- * the operation performed for this callback
- * @param primaryKeyFields
- * the indexes of the primary keys (used for lock operations)
- * @return
- * an instance of {@link org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory}
- * @throws AlgebricksException
- * If the callback factory could not be created
+ * @param index the index
+ * @param op the operation performed for this callback
+ * @param primaryKeyFields the indexes of the primary keys (used for lock operations)
+ * @return an instance of {@link org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory}
+ * @throws AlgebricksException If the callback factory could not be created
*/
public IModificationOperationCallbackFactory getModificationCallbackFactory(
IStorageComponentProvider componentProvider, Index index, IndexOperation op, int[] primaryKeyFields)
@@ -680,6 +648,7 @@
tree.put("pendingOp", MetadataUtil.pendingOpToString(pendingOp));
tree.put("rebalanceCount", rebalanceCount);
tree.put("compressionScheme", compressionScheme);
+ tree.put("datasetFormatInfo", datasetFormatInfo.toString());
return tree;
}
@@ -691,12 +660,9 @@
/**
* Gets the commit runtime factory for inserting/upserting/deleting operations on this dataset.
*
- * @param metadataProvider,
- * the metadata provider.
- * @param primaryKeyFieldPermutation,
- * the primary key field permutation according to the input.
- * @param isSink,
- * whether this commit runtime is the last operator in the pipeline.
+ * @param metadataProvider, the metadata provider.
+ * @param primaryKeyFieldPermutation, the primary key field permutation according to the input.
+ * @param isSink, whether this commit runtime is the last operator in the pipeline.
* @return the commit runtime factory for inserting/upserting/deleting operations on this dataset.
* @throws AlgebricksException
*/
@@ -724,10 +690,10 @@
}
/**
- @return the array of type traits that contains the following type traits in order
- 1) the primary keys,
- 2) the query record type,
- 3) the metadata type trait if the dataset has metadata
+ * @return the array of type traits that contains the following type traits in order
+ * 1) the primary keys,
+ * 2) the query record type,
+ * 3) the metadata type trait if the dataset has metadata
*/
// ToDo: returning such an array can be confusing because it may contain the metadata type or not.
// instead of returning an array, create a new class that contains 1) a type trait array for the primary keys,
@@ -760,8 +726,7 @@
/**
* Gets the record descriptor for primary records of this dataset.
*
- * @param metadataProvider,
- * the metadata provider.
+ * @param metadataProvider, the metadata provider.
* @return the record descriptor for primary records of this dataset.
* @throws AlgebricksException
*/
@@ -805,8 +770,7 @@
/**
* Gets the comparator factories for the primary key fields of this dataset.
*
- * @param metadataProvider,
- * the metadata provider.
+ * @param metadataProvider, the metadata provider.
* @return the comparator factories for the primary key fields of this dataset.
* @throws AlgebricksException
*/
@@ -834,8 +798,7 @@
/**
* Gets the hash function factories for the primary key fields of this dataset.
*
- * @param metadataProvider,
- * the metadata provider.
+ * @param metadataProvider, the metadata provider.
* @return the hash function factories for the primary key fields of this dataset.
* @throws AlgebricksException
*/
@@ -873,7 +836,7 @@
this.metaTypeDataverseName, this.metaTypeName, targetNodeGroupName, this.compactionPolicyFactory,
this.compactionPolicyProperties, this.datasetDetails, this.hints, this.datasetType,
DatasetIdFactory.generateAlternatingDatasetId(this.datasetId), this.pendingOp, this.rebalanceCount + 1,
- this.compressionScheme);
+ this.compressionScheme, this.datasetFormatInfo);
}
// Gets an array of partition numbers for this dataset.
@@ -895,4 +858,7 @@
return datasetFullyQualifiedName;
}
+ public DatasetFormatInfo getDatasetFormatInfo() {
+ return datasetFormatInfo;
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
index 3f4e63e..1ae2430 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
@@ -35,6 +35,7 @@
import org.apache.asterix.metadata.api.IMetadataIndex;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AInt64;
import org.apache.asterix.om.base.AMutableString;
@@ -77,6 +78,9 @@
@SuppressWarnings("unchecked")
protected final ISerializerDeserializer<AInt64> int64Serde =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
+ @SuppressWarnings("unchecked")
+ protected final ISerializerDeserializer<ADouble> doubleSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);
protected final ISerializerDeserializer<ARecord> recordSerDes;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 0d05d46..eafa331 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -33,6 +33,7 @@
import org.apache.asterix.builders.OrderedListBuilder;
import org.apache.asterix.builders.RecordBuilder;
import org.apache.asterix.builders.UnorderedListBuilder;
+import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.TransactionState;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -42,6 +43,7 @@
import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
@@ -51,9 +53,11 @@
import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.ADateTime;
+import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AInt64;
import org.apache.asterix.om.base.AInt8;
+import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
import org.apache.asterix.om.base.AMutableString;
@@ -88,12 +92,14 @@
protected AMutableInt32 aInt32;
protected AMutableInt64 aInt64;
+ protected AMutableDouble aDouble;
protected DatasetTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.DATASET_DATASET, DATASET_PAYLOAD_TUPLE_FIELD_INDEX);
if (getTuple) {
aInt32 = new AMutableInt32(-1);
aInt64 = new AMutableInt64(-1);
+ aDouble = new AMutableDouble(0.0);
}
}
@@ -369,10 +375,11 @@
long rebalanceCount = getRebalanceCount(datasetRecord);
String compressionScheme = getCompressionScheme(datasetRecord);
+ DatasetFormatInfo datasetFormatInfo = getDatasetFormatInfo(datasetRecord);
return new Dataset(dataverseName, datasetName, typeDataverseName, typeName, metaTypeDataverseName, metaTypeName,
nodeGroupName, compactionPolicy.first, compactionPolicy.second, datasetDetails, hints, datasetType,
- datasetId, pendingOp, rebalanceCount, compressionScheme);
+ datasetId, pendingOp, rebalanceCount, compressionScheme, datasetFormatInfo);
}
protected Pair<String, Map<String, String>> readCompactionPolicy(DatasetType datasetType, ARecord datasetRecord) {
@@ -421,6 +428,39 @@
return CompressionManager.NONE;
}
+ private DatasetFormatInfo getDatasetFormatInfo(ARecord datasetRecord) {
+ ARecordType datasetType = datasetRecord.getType();
+ int datasetFormatIndex =
+ datasetType.getFieldIndex(MetadataRecordTypes.DATASET_ARECORD_DATASET_FORMAT_FIELD_NAME);
+ if (datasetFormatIndex < 0) {
+ return DatasetFormatInfo.DEFAULT;
+ }
+
+ ARecordType datasetFormatType = (ARecordType) datasetType.getFieldTypes()[datasetFormatIndex];
+ int formatIndex =
+ datasetFormatType.getFieldIndex(MetadataRecordTypes.DATASET_ARECORD_DATASET_FORMAT_FORMAT_FIELD_NAME);
+ int maxTupleCountIndex =
+ datasetFormatType.getFieldIndex(MetadataRecordTypes.DATASET_ARECORD_DATASET_MAX_TUPLE_COUNT_FIELD_NAME);
+ int freeSpaceToleranceIndex = datasetFormatType
+ .getFieldIndex(MetadataRecordTypes.DATASET_ARECORD_DATASET_FREE_SPACE_TOLERANCE_FIELD_NAME);
+
+ ARecord datasetFormatRecord = (ARecord) datasetRecord.getValueByPos(datasetFormatIndex);
+
+ //Format
+ AString formatString = (AString) datasetFormatRecord.getValueByPos(formatIndex);
+ DatasetConfig.DatasetFormat format = DatasetConfig.DatasetFormat.valueOf(formatString.getStringValue());
+
+ //MaxTupleCount
+ AInt64 maxTupleCountInt = (AInt64) datasetFormatRecord.getValueByPos(maxTupleCountIndex);
+ int maxTupleCount = (int) maxTupleCountInt.getLongValue();
+
+ //FreeSpaceTolerance
+ ADouble freeSpaceToleranceDouble = (ADouble) datasetFormatRecord.getValueByPos(freeSpaceToleranceIndex);
+ float freeSpaceTolerance = (float) freeSpaceToleranceDouble.getDoubleValue();
+
+ return new DatasetFormatInfo(format, maxTupleCount, freeSpaceTolerance);
+ }
+
@Override
public ITupleReference getTupleFromMetadataEntity(Dataset dataset) throws HyracksDataException {
OrderedListBuilder listBuilder = new OrderedListBuilder();
@@ -575,6 +615,7 @@
writeRebalanceCount(dataset);
writeBlockLevelStorageCompression(dataset);
writeOpenDetails(dataset);
+ writeDatasetFormatInfo(dataset);
}
private void writeOpenDetails(Dataset dataset) throws HyracksDataException {
@@ -633,6 +674,47 @@
recordBuilder.addField(fieldName, fieldValue);
}
+ private void writeDatasetFormatInfo(Dataset dataset) throws HyracksDataException {
+ DatasetFormatInfo info = dataset.getDatasetFormatInfo();
+ if (DatasetFormatInfo.DEFAULT == info) {
+ return;
+ }
+
+ RecordBuilder datasetFormatObject = new RecordBuilder();
+ datasetFormatObject.reset(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+
+ fieldName.reset();
+ aString.setValue(MetadataRecordTypes.DATASET_ARECORD_DATASET_FORMAT_FORMAT_FIELD_NAME);
+ stringSerde.serialize(aString, fieldName.getDataOutput());
+ fieldValue.reset();
+ aString.setValue(info.getFormat().toString());
+ stringSerde.serialize(aString, fieldValue.getDataOutput());
+ datasetFormatObject.addField(fieldName, fieldValue);
+
+ fieldName.reset();
+ aString.setValue(MetadataRecordTypes.DATASET_ARECORD_DATASET_MAX_TUPLE_COUNT_FIELD_NAME);
+ stringSerde.serialize(aString, fieldName.getDataOutput());
+ fieldValue.reset();
+ aInt64.setValue(info.getMaxTupleCount());
+ int64Serde.serialize(aInt64, fieldValue.getDataOutput());
+ datasetFormatObject.addField(fieldName, fieldValue);
+
+ fieldName.reset();
+ aString.setValue(MetadataRecordTypes.DATASET_ARECORD_DATASET_FREE_SPACE_TOLERANCE_FIELD_NAME);
+ stringSerde.serialize(aString, fieldName.getDataOutput());
+ fieldValue.reset();
+ aDouble.setValue(info.getFreeSpaceTolerance());
+ doubleSerde.serialize(aDouble, fieldValue.getDataOutput());
+ datasetFormatObject.addField(fieldName, fieldValue);
+
+ fieldName.reset();
+ aString.setValue(MetadataRecordTypes.DATASET_ARECORD_DATASET_FORMAT_FIELD_NAME);
+ stringSerde.serialize(aString, fieldName.getDataOutput());
+ fieldValue.reset();
+ datasetFormatObject.write(fieldValue.getDataOutput(), true);
+ recordBuilder.addField(fieldName, fieldValue);
+ }
+
private void writeRebalanceCount(Dataset dataset) throws HyracksDataException {
if (dataset.getRebalanceCount() > 0) {
// Adds the field rebalanceCount.
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 5fd96a6..ce98008 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -174,7 +174,7 @@
if (fnKind != FunctionKind.SCALAR) {
return false;
}
- if (!(fnInfo instanceof IExternalFunctionInfo)) {
+ if (!fnInfo.isExternal()) {
return false;
}
ExternalFunctionLanguage language = ((IExternalFunctionInfo) fnInfo).getLanguage();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ColumnFilterBuilder.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ColumnFilterBuilder.java
new file mode 100644
index 0000000..e8c2b7f
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ColumnFilterBuilder.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.utils;
+
+import static org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterValueAccessorFactory;
+import org.apache.asterix.column.values.reader.filter.compartor.GEColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.compartor.GTColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.compartor.LEColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.compartor.LTColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.evaluator.ANDColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.evaluator.NoOpColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.evaluator.ORColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.value.ColumnFilterValueAccessorFactory;
+import org.apache.asterix.column.values.reader.filter.value.ConstantColumnFilterValueAccessorFactory;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+
+public class ColumnFilterBuilder {
+ public static final Set<FunctionIdentifier> COMPARE_FUNCTIONS = getCompareFunctions();
+ public static final Set<FunctionIdentifier> PUSHABLE_FUNCTIONS = getPushableFunctions();
+
+ private final Map<ILogicalExpression, ARecordType> filterPaths;
+ private final ILogicalExpression filterExpression;
+
+ public ColumnFilterBuilder(DataProjectionFiltrationInfo projectionFiltrationInfo) {
+ this.filterPaths = projectionFiltrationInfo.getExpressionToPath();
+ this.filterExpression = projectionFiltrationInfo.getFilterExpression();
+ }
+
+ public IColumnFilterEvaluatorFactory build() {
+ if (filterExpression == null || filterPaths.isEmpty()) {
+ return NoOpColumnFilterEvaluatorFactory.INSTANCE;
+ }
+ return createEvaluator(filterPaths, filterExpression);
+ }
+
+ private IColumnFilterEvaluatorFactory createEvaluator(Map<ILogicalExpression, ARecordType> filterPaths,
+ ILogicalExpression filterExpression) {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) filterExpression;
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ if (COMPARE_FUNCTIONS.contains(fid)) {
+ return createComparator(fid, funcExpr.getArguments(), filterPaths);
+ }
+ return createEvaluatorsForArgs(funcExpr, filterPaths);
+ }
+
+ private IColumnFilterEvaluatorFactory createComparator(FunctionIdentifier fid,
+ List<Mutable<ILogicalExpression>> arguments, Map<ILogicalExpression, ARecordType> filterPaths) {
+ ILogicalExpression arg0 = arguments.get(0).getValue();
+ ILogicalExpression arg1 = arguments.get(1).getValue();
+
+ if (arg1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ ARecordType path = filterPaths.get(arg0);
+ IAObject constant = getConstant(arg1);
+ return createComparator(fid, path, constant, true);
+ } else {
+ ARecordType path = filterPaths.get(arg1);
+ IAObject constant = getConstant(arg0);
+ return createComparator(fid, path, constant, false);
+ }
+ }
+
+ private IColumnFilterEvaluatorFactory createEvaluatorsForArgs(AbstractFunctionCallExpression funcExpr,
+ Map<ILogicalExpression, ARecordType> filterPaths) {
+ List<Mutable<ILogicalExpression>> args = funcExpr.getArguments();
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ //AND/OR have at least two arguments
+ IColumnFilterEvaluatorFactory arg0 = createEvaluator(filterPaths, args.get(0).getValue());
+ IColumnFilterEvaluatorFactory arg1 = createEvaluator(filterPaths, args.get(1).getValue());
+
+ IColumnFilterEvaluatorFactory left = createEvaluator(fid, arg0, arg1);
+ for (int i = 2; i < args.size() && left != null; i++) {
+ IColumnFilterEvaluatorFactory right = createEvaluator(filterPaths, args.get(i).getValue());
+ left = createEvaluator(fid, left, right);
+ }
+ return left;
+ }
+
+ private IColumnFilterEvaluatorFactory createComparator(FunctionIdentifier fid, ARecordType path, IAObject constant,
+ boolean leftToRight) {
+ ComparisonKind comparisonKind = AlgebricksBuiltinFunctions.getComparisonType(fid);
+
+ IColumnFilterValueAccessorFactory constValue = ConstantColumnFilterValueAccessorFactory.createFactory(constant);
+ IColumnFilterValueAccessorFactory min = new ColumnFilterValueAccessorFactory(path, true);
+ IColumnFilterValueAccessorFactory max = new ColumnFilterValueAccessorFactory(path, false);
+
+ if (leftToRight) {
+ return createEvaluator(comparisonKind, min, constValue, max);
+ }
+ return createEvaluator(invert(comparisonKind), min, constValue, max);
+ }
+
+ private static IColumnFilterEvaluatorFactory createEvaluator(FunctionIdentifier fid,
+ IColumnFilterEvaluatorFactory left, IColumnFilterEvaluatorFactory right) {
+ if (right == null) {
+ return null;
+ }
+ if (BuiltinFunctions.AND.equals(fid)) {
+ return new ANDColumnFilterEvaluatorFactory(left, right);
+ }
+ return new ORColumnFilterEvaluatorFactory(left, right);
+ }
+
+ private static ComparisonKind invert(ComparisonKind comparisonKind) {
+ if (comparisonKind == ComparisonKind.LT) {
+ return ComparisonKind.GE;
+ } else if (comparisonKind == ComparisonKind.LE) {
+ return ComparisonKind.GT;
+ } else if (comparisonKind == ComparisonKind.GT) {
+ return ComparisonKind.LE;
+ }
+ //ComparisonKind.GE
+ return ComparisonKind.LT;
+ }
+
+ private static IColumnFilterEvaluatorFactory createEvaluator(ComparisonKind comparisonKind,
+ IColumnFilterValueAccessorFactory min, IColumnFilterValueAccessorFactory constVal,
+ IColumnFilterValueAccessorFactory max) {
+ if (comparisonKind == ComparisonKind.LT) {
+ return new GTColumnFilterEvaluatorFactory(constVal, min);
+ } else if (comparisonKind == ComparisonKind.LE) {
+ return new GEColumnFilterEvaluatorFactory(constVal, min);
+ } else if (comparisonKind == ComparisonKind.EQ) {
+ IColumnFilterEvaluatorFactory minComp = new GEColumnFilterEvaluatorFactory(constVal, min);
+ IColumnFilterEvaluatorFactory maxComp = new LEColumnFilterEvaluatorFactory(constVal, max);
+ return new ANDColumnFilterEvaluatorFactory(minComp, maxComp);
+ } else if (comparisonKind == ComparisonKind.GT) {
+ return new LTColumnFilterEvaluatorFactory(constVal, max);
+ }
+ //ComparisonKind.GE
+ return new LEColumnFilterEvaluatorFactory(constVal, max);
+ }
+
+ private static IAObject getConstant(ILogicalExpression expr) {
+ return ((AsterixConstantValue) ((ConstantExpression) expr).getValue()).getObject();
+ }
+
+ private static Set<FunctionIdentifier> getCompareFunctions() {
+ return Set.of(AlgebricksBuiltinFunctions.LE, AlgebricksBuiltinFunctions.GE, AlgebricksBuiltinFunctions.LT,
+ AlgebricksBuiltinFunctions.GT, AlgebricksBuiltinFunctions.EQ);
+ }
+
+ private static Set<FunctionIdentifier> getPushableFunctions() {
+ Set<FunctionIdentifier> pushableFunctions = new HashSet<>(COMPARE_FUNCTIONS);
+ pushableFunctions.add(AlgebricksBuiltinFunctions.AND);
+ pushableFunctions.add(AlgebricksBuiltinFunctions.OR);
+ return pushableFunctions;
+ }
+
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index 7e9cfc3..fc8fdb6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -20,31 +20,43 @@
import static org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import org.apache.asterix.column.operation.query.QueryColumnTupleProjectorFactory;
+import org.apache.asterix.column.values.reader.filter.IColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.values.reader.filter.evaluator.NoOpColumnFilterEvaluatorFactory;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
+import org.apache.asterix.runtime.projection.DataProjectionFiltrationInfo;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.job.IJobletEventListenerFactory;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
import org.apache.hyracks.util.OptionalBoolean;
public class IndexUtil {
@@ -179,10 +191,8 @@
/**
* Binds a job event listener to the job specification.
*
- * @param spec,
- * the job specification.
- * @param metadataProvider,
- * the metadata provider.
+ * @param spec, the job specification.
+ * @param metadataProvider, the metadata provider.
* @return the AsterixDB job id for transaction management.
*/
public static void bindJobEventListener(JobSpecification spec, MetadataProvider metadataProvider)
@@ -253,4 +263,38 @@
return new Pair<>(MetadataConstants.SAMPLE_INDEX_1_PREFIX + datasetName,
MetadataConstants.SAMPLE_INDEX_2_PREFIX + datasetName);
}
+
+ public static ITupleProjectorFactory createTupleProjectorFactory(DatasetFormatInfo datasetFormatInfo,
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo,
+ ARecordType datasetType, ARecordType metaItemType, int numberOfPrimaryKeys) {
+ if (datasetFormatInfo.getFormat() == DatasetConfig.DatasetFormat.ROW) {
+ return DefaultTupleProjectorFactory.INSTANCE;
+ }
+ DataProjectionFiltrationInfo dataProjectionInfo = (DataProjectionFiltrationInfo) projectionInfo;
+ if (dataProjectionInfo == null) {
+ // projecting pushdown is disabled
+ ARecordType metaType = metaItemType == null ? null : DataProjectionFiltrationInfo.ALL_FIELDS_TYPE;
+ return new QueryColumnTupleProjectorFactory(datasetType, metaItemType, numberOfPrimaryKeys,
+ DataProjectionFiltrationInfo.ALL_FIELDS_TYPE, Collections.emptyMap(), metaType,
+ Collections.emptyMap(), NoOpColumnFilterEvaluatorFactory.INSTANCE);
+ }
+
+ DataProjectionFiltrationInfo metaDataProjectionInfo = (DataProjectionFiltrationInfo) metaProjectionInfo;
+
+ ARecordType datasetRequestedType = dataProjectionInfo.getProjectionInfo();
+ Map<String, FunctionCallInformation> datasetFunctionCallInfo = dataProjectionInfo.getFunctionCallInfoMap();
+
+ ARecordType metaRequestedType =
+ metaDataProjectionInfo == null ? null : metaDataProjectionInfo.getProjectionInfo();
+ Map<String, FunctionCallInformation> metaFunctionCallInfo =
+ metaProjectionInfo == null ? null : metaDataProjectionInfo.getFunctionCallInfoMap();
+
+ ColumnFilterBuilder columnFilterBuilder = new ColumnFilterBuilder(dataProjectionInfo);
+ IColumnFilterEvaluatorFactory filterEvaluator = columnFilterBuilder.build();
+
+ return new QueryColumnTupleProjectorFactory(datasetType, metaItemType, numberOfPrimaryKeys,
+ datasetRequestedType, datasetFunctionCallInfo, metaRequestedType, metaFunctionCallInfo,
+ filterEvaluator);
+ }
+
}
diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
index 73eeae4..292ff16 100644
--- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
+++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
@@ -25,6 +25,7 @@
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
@@ -55,7 +56,8 @@
Dataset dataset = new Dataset(DataverseName.createSinglePartName("test"), "log",
DataverseName.createSinglePartName("foo"), "LogType", DataverseName.createSinglePartName("CB"),
"MetaType", "DEFAULT_NG_ALL_NODES", "prefix", compactionPolicyProperties, details,
- Collections.emptyMap(), DatasetType.INTERNAL, 115, 0, CompressionManager.NONE);
+ Collections.emptyMap(), DatasetType.INTERNAL, 115, 0, CompressionManager.NONE,
+ DatasetFormatInfo.DEFAULT);
DatasetTupleTranslator dtTranslator = new DatasetTupleTranslator(true);
ITupleReference tuple = dtTranslator.getTupleFromMetadataEntity(dataset);
Dataset deserializedDataset = dtTranslator.getMetadataEntityFromTuple(tuple);
@@ -63,7 +65,7 @@
deserializedDataset.getMetaItemTypeDataverseName());
Assert.assertEquals(dataset.getMetaItemTypeName(), deserializedDataset.getMetaItemTypeName());
if (indicator == null) {
- Assert.assertEquals(Collections.singletonList(new Integer(0)),
+ Assert.assertEquals(Collections.singletonList(Integer.valueOf(0)),
((InternalDatasetDetails) deserializedDataset.getDatasetDetails()).getKeySourceIndicator());
} else {
Assert.assertEquals(((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(),
diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
index edb360f..77c64d7 100644
--- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
+++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
@@ -32,6 +32,7 @@
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.metadata.MetadataNode;
+import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Index;
@@ -69,7 +70,7 @@
DataverseName dvCB = DataverseName.createSinglePartName("CB");
Dataset dataset = new Dataset(dvTest, "d1", dvFoo, "LogType", dvCB, "MetaType", "DEFAULT_NG_ALL_NODES",
"prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115, 0,
- CompressionManager.NONE);
+ CompressionManager.NONE, DatasetFormatInfo.DEFAULT);
Index index = new Index(dvTest, "d1", "i1", IndexType.BTREE,
Collections.singletonList(Collections.singletonList("row_id")),
diff --git a/asterixdb/asterix-om/pom.xml b/asterixdb/asterix-om/pom.xml
index 0fed772..2ada291 100644
--- a/asterixdb/asterix-om/pom.xml
+++ b/asterixdb/asterix-om/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-om</artifactId>
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AUUIDPartialBinaryComparatorFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AUUIDPartialBinaryComparatorFactory.java
index 01f2537..d842c89 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AUUIDPartialBinaryComparatorFactory.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AUUIDPartialBinaryComparatorFactory.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.LongPointable;
import com.fasterxml.jackson.databind.JsonNode;
@@ -37,6 +38,11 @@
return AUUIDPartialBinaryComparatorFactory::compare;
}
+ public static int compare(IValueReference valueA, IValueReference valueB) {
+ return compare(valueA.getByteArray(), valueA.getStartOffset(), valueA.getLength(), valueB.getByteArray(),
+ valueB.getStartOffset(), valueB.getLength());
+ }
+
@SuppressWarnings("squid:S1172") // unused parameter
public static int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
int msbCompare = Long.compare(LongPointable.getLong(b1, s1), LongPointable.getLong(b2, s2));
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/object/base/AdmObjectNode.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/object/base/AdmObjectNode.java
index 294f48b..966b9ba 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/object/base/AdmObjectNode.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/object/base/AdmObjectNode.java
@@ -162,6 +162,22 @@
return ((AdmStringNode) node).get();
}
+ public int getOptionalInt(String field, int defaultValue) {
+ final IAdmNode node = get(field);
+ if (node == null) {
+ return defaultValue;
+ }
+ return (int) ((AdmBigIntNode) node).get();
+ }
+
+ public float getOptionalFloat(String field, float defaultValue) {
+ final IAdmNode node = get(field);
+ if (node == null) {
+ return defaultValue;
+ }
+ return (float) ((AdmDoubleNode) node).get();
+ }
+
public static String getString(AdmObjectNode openFields, String field) throws HyracksDataException {
IAdmNode node = openFields.get(field);
if (node == null) {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index 4db68c3..cdc8111 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -119,8 +119,12 @@
import org.apache.asterix.om.typecomputer.impl.OrderedListOfAnyTypeComputer;
import org.apache.asterix.om.typecomputer.impl.PropagateTypeComputer;
import org.apache.asterix.om.typecomputer.impl.RecordAddFieldsTypeComputer;
+import org.apache.asterix.om.typecomputer.impl.RecordAddTypeComputer;
import org.apache.asterix.om.typecomputer.impl.RecordMergeTypeComputer;
+import org.apache.asterix.om.typecomputer.impl.RecordPutTypeComputer;
import org.apache.asterix.om.typecomputer.impl.RecordRemoveFieldsTypeComputer;
+import org.apache.asterix.om.typecomputer.impl.RecordRemoveTypeComputer;
+import org.apache.asterix.om.typecomputer.impl.RecordRenameTypeComputer;
import org.apache.asterix.om.typecomputer.impl.ScalarArrayAggTypeComputer;
import org.apache.asterix.om.typecomputer.impl.ScalarVersionOfAggregateResultType;
import org.apache.asterix.om.typecomputer.impl.SleepTypeComputer;
@@ -250,6 +254,12 @@
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "array-slice", 3);
public static final FunctionIdentifier ARRAY_EXCEPT =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "array-except", 2);
+ public static final FunctionIdentifier ARRAY_SWAP =
+ new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "array-swap", 3);
+ public static final FunctionIdentifier ARRAY_MOVE =
+ new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "array-move", 3);
+ public static final FunctionIdentifier ARRAY_BINARY_SEARCH =
+ new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "array-binary-search", 2);
// objects
public static final FunctionIdentifier RECORD_MERGE =
@@ -2408,6 +2418,9 @@
addFunction(ARRAY_SLICE_WITH_END_POSITION, AListTypeComputer.INSTANCE_SLICE, true);
addFunction(ARRAY_SLICE_WITHOUT_END_POSITION, AListTypeComputer.INSTANCE_SLICE, true);
addFunction(ARRAY_EXCEPT, ArrayExceptTypeComputer.INSTANCE, true);
+ addFunction(ARRAY_MOVE, AListTypeComputer.INSTANCE_MOVE, true);
+ addFunction(ARRAY_SWAP, AListTypeComputer.INSTANCE_SWAP, true);
+ addFunction(ARRAY_BINARY_SEARCH, AInt32TypeComputer.INSTANCE_NULLABLE, true);
// objects
addFunction(RECORD_MERGE, RecordMergeTypeComputer.INSTANCE, true);
@@ -2428,12 +2441,12 @@
addFunction(RECORD_PAIRS, OrderedListOfAnyTypeComputer.INSTANCE_NULLABLE, true);
addFunction(PAIRS, OrderedListOfAnyTypeComputer.INSTANCE_NULLABLE, true);
addFunction(GEOMETRY_CONSTRUCTOR, AGeometryTypeComputer.INSTANCE, true);
- addFunction(RECORD_REMOVE, OpenARecordTypeComputer.INSTANCE, true);
- addFunction(RECORD_RENAME, OpenARecordTypeComputer.INSTANCE, true);
+ addFunction(RECORD_REMOVE, RecordRemoveTypeComputer.INSTANCE, true);
+ addFunction(RECORD_RENAME, RecordRenameTypeComputer.INSTANCE, true);
addFunction(RECORD_UNWRAP, AnyTypeComputer.INSTANCE, true);
addFunction(RECORD_REPLACE, OpenARecordTypeComputer.INSTANCE, true);
- addFunction(RECORD_ADD, OpenARecordTypeComputer.INSTANCE, true);
- addFunction(RECORD_PUT, OpenARecordTypeComputer.INSTANCE, true);
+ addFunction(RECORD_ADD, RecordAddTypeComputer.INSTANCE, true);
+ addFunction(RECORD_PUT, RecordPutTypeComputer.INSTANCE, true);
addFunction(RECORD_VALUES, OrderedListOfAnyTypeComputer.INSTANCE, true);
// temporal type accessors
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
index a3ec9c6..8a2cefa 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
@@ -101,4 +101,9 @@
public boolean getNullCall() {
return nullCall;
}
+
+ @Override
+ public boolean isExternal() {
+ return true;
+ }
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractLazyNestedVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractLazyNestedVisitablePointable.java
new file mode 100644
index 0000000..e9f8e8a
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractLazyNestedVisitablePointable.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+/**
+ * A common implementation for nested values (i.e., {@link ATypeTag#OBJECT}, {@link ATypeTag#ARRAY}, and
+ * {@link ATypeTag#MULTISET}).
+ * <p>
+ * Contract:
+ * <p>
+ * 1- A child's value may or may not contain a type tag. Thus, it is the responsibility of the caller to check if the
+ * child's value contains a type tag by calling {@link #isTaggedChild()}.
+ * 2- The returned objects from {@link #getChildVisitablePointable()} and {@link #getChildValue()}, are reused
+ * when possible. Thus, when the caller does the following for example:
+ * <p>
+ * AbstractLazyVisitablePointable child1 = visitablePointable.getChildVisitablePointable();
+ * visitablePointable.nextChild();
+ * AbstractLazyVisitablePointable child2 = visitablePointable.getChildVisitablePointable();
+ * <p>
+ * both child1 and child2 may have the same value, which is the value of the second child.
+ */
+public abstract class AbstractLazyNestedVisitablePointable extends AbstractLazyVisitablePointable {
+ private final ATypeTag typeTag;
+ protected final VoidPointable currentValue;
+ protected byte currentChildTypeTag;
+
+ AbstractLazyNestedVisitablePointable(boolean tagged, ATypeTag typeTag) {
+ super(tagged);
+ this.typeTag = typeTag;
+ currentValue = new VoidPointable();
+ }
+
+ /**
+ * Prepare the value and the tag of the next child
+ */
+ public abstract void nextChild() throws HyracksDataException;
+
+ /**
+ * If the child contains a tag
+ *
+ * @return true if the child is tagged (open value), false otherwise
+ */
+ public abstract boolean isTaggedChild();
+
+ /**
+ * @return number of children
+ */
+ public abstract int getNumberOfChildren();
+
+ /**
+ * Gets a child visitable-pointable.
+ */
+ public abstract AbstractLazyVisitablePointable getChildVisitablePointable() throws HyracksDataException;
+
+ /**
+ * Returns a value reference of the child. Note that this is not a visitable-pointable reference.
+ */
+ public final IValueReference getChildValue() {
+ return currentValue;
+ }
+
+ /**
+ * The serialized type tag of a child
+ */
+ public final byte getChildSerializedTypeTag() {
+ return currentChildTypeTag;
+ }
+
+ /**
+ * The type tag of a child
+ */
+ public final ATypeTag getChildTypeTag() {
+ return ATypeTag.VALUE_TYPE_MAPPING[currentChildTypeTag];
+ }
+
+ /**
+ * @return The type tag that corresponds to {@code this} visitable-pointable
+ */
+ @Override
+ public final ATypeTag getTypeTag() {
+ return ATypeTag.VALUE_TYPE_MAPPING[getSerializedTypeTag()];
+ }
+
+ /**
+ * @return The serialized type tag that corresponds to {@code this} visitable-pointable
+ */
+ @Override
+ public final byte getSerializedTypeTag() {
+ return typeTag.serialize();
+ }
+
+ /**
+ * Helper method to create a typed (i.e., non-tagged) visitable-pointable
+ *
+ * @param type the required type
+ * @return a visitable pointable that corresponds to {@code type}
+ */
+ static AbstractLazyVisitablePointable createVisitable(IAType type) {
+ ATypeTag typeTag = type.getTypeTag();
+ switch (typeTag) {
+ case OBJECT:
+ return new TypedRecordLazyVisitablePointable(false, (ARecordType) type);
+ case ARRAY:
+ case MULTISET:
+ AbstractCollectionType listType = (AbstractCollectionType) type;
+ return NonTaggedFormatUtil.isFixedSizedCollection(listType.getItemType())
+ ? new FixedListLazyVisitablePointable(false, listType)
+ : new VariableListLazyVisitablePointable(false, listType);
+ default:
+ return new FlatLazyVisitablePointable(false, typeTag);
+
+ }
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractLazyVisitablePointable.java
new file mode 100644
index 0000000..1d6425c
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractLazyVisitablePointable.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+/**
+ * Similar to {@link IVisitablePointable}. The difference is, upon calling {@link #set(byte[], int, int)}, the nested
+ * values (children) will not call {@link #set(byte[], int, int)} recursively. Instead, it will wait until the
+ * child is accessed. Thus, when a processor (a class that implements {@link ILazyVisitablePointableVisitor}) wants
+ * to traverse the object in a <b>DFS mode</b>, the traversal will be done in a single pass - compared to the two passes
+ * when using the {@link IVisitablePointable}, where one pass is done when calling
+ * {@link IVisitablePointable#set(byte[], int, int)} and another pass is done by the processor (e.g., the result
+ * printer). Also, the lazy visitable-pointable requires less memory as we do not allocate any temporary buffers.
+ */
+public abstract class AbstractLazyVisitablePointable implements IPointable {
+ private final boolean tagged;
+ private byte[] data;
+ private int offset;
+ private int length;
+
+ AbstractLazyVisitablePointable(boolean tagged) {
+ this.tagged = tagged;
+ }
+
+ @Override
+ public final void set(byte[] data, int offset, int length) {
+ this.data = data;
+ this.offset = offset;
+ this.length = length;
+ init(data, offset, length);
+ }
+
+ @Override
+ public final void set(IValueReference pointer) {
+ set(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public final byte[] getByteArray() {
+ return data;
+ }
+
+ @Override
+ public final int getStartOffset() {
+ return offset;
+ }
+
+ @Override
+ public final int getLength() {
+ return length;
+ }
+
+ /**
+ * @return The serialized type tag
+ */
+ public abstract byte getSerializedTypeTag();
+
+ /**
+ * @return The type tag
+ */
+ public abstract ATypeTag getTypeTag();
+
+ public abstract <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException;
+
+ /**
+ * @return true if the value contains tag, false otherwise
+ */
+ public final boolean isTagged() {
+ return tagged;
+ }
+
+ /**
+ * Called by {@link #set(byte[], int, int)} to initialize the visitable-pointable
+ *
+ * @param data value's data
+ * @param offset value's start offset
+ * @param length value's length
+ */
+ abstract void init(byte[] data, int offset, int length);
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractListLazyVisitablePointable.java
new file mode 100644
index 0000000..68d558c
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/AbstractListLazyVisitablePointable.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Common implementation for both {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET}
+ */
+public abstract class AbstractListLazyVisitablePointable extends AbstractLazyNestedVisitablePointable {
+ private final int headerSize;
+ private final AbstractLazyVisitablePointable itemVisitablePointable;
+ private int numberOfItems;
+ protected int currentIndex;
+ protected int itemsOffset;
+
+ AbstractListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+ super(tagged, listType.getTypeTag());
+ Objects.requireNonNull(listType);
+ Objects.requireNonNull(listType.getItemType());
+ //1 for typeTag if tagged, 1 for itemTypeTag, 4 for length
+ headerSize = (isTagged() ? 1 : 0) + 1 + 4;
+ itemVisitablePointable = createVisitablePointable(listType.getItemType());
+ }
+
+ @Override
+ public final int getNumberOfChildren() {
+ return numberOfItems;
+ }
+
+ @Override
+ final void init(byte[] data, int offset, int length) {
+ int pointer = headerSize + offset;
+ numberOfItems = AInt32SerializerDeserializer.getInt(data, pointer);
+ itemsOffset = pointer + 4;
+ currentIndex = 0;
+ }
+
+ @Override
+ public AbstractLazyVisitablePointable getChildVisitablePointable() throws HyracksDataException {
+ itemVisitablePointable.set(getChildValue());
+ return itemVisitablePointable;
+ }
+
+ @Override
+ public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ return visitor.visit(this, arg);
+ }
+
+ abstract AbstractLazyVisitablePointable createVisitablePointable(IAType itemType);
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/FixedListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/FixedListLazyVisitablePointable.java
new file mode 100644
index 0000000..f3153b2
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/FixedListLazyVisitablePointable.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This implementation is only for closed {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET} with fixed-length
+ * items.
+ */
+public class FixedListLazyVisitablePointable extends AbstractListLazyVisitablePointable {
+ private final int itemSize;
+
+ public FixedListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+ super(tagged, listType);
+ ATypeTag itemTag = listType.getItemType().getTypeTag();
+ currentChildTypeTag = itemTag.serialize();
+ try {
+ itemSize = NonTaggedFormatUtil.getFieldValueLength(null, -1, itemTag, false);
+ } catch (HyracksDataException e) {
+ throw new IllegalStateException(e);
+ }
+
+ }
+
+ @Override
+ public void nextChild() {
+ byte[] data = getByteArray();
+ int itemOffset = getStartOffset() + itemsOffset + currentIndex * itemSize;
+ currentValue.set(data, itemOffset, itemSize);
+ currentIndex++;
+ }
+
+ @Override
+ public boolean isTaggedChild() {
+ return false;
+ }
+
+ @Override
+ AbstractLazyVisitablePointable createVisitablePointable(IAType itemType) {
+ return createVisitable(itemType);
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/FlatLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/FlatLazyVisitablePointable.java
new file mode 100644
index 0000000..f441c7b
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/FlatLazyVisitablePointable.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FlatLazyVisitablePointable extends AbstractLazyVisitablePointable {
+ private final ATypeTag typeTag;
+
+ public FlatLazyVisitablePointable(boolean tagged, ATypeTag typeTag) {
+ super(tagged);
+ Objects.requireNonNull(typeTag);
+ this.typeTag = typeTag;
+ }
+
+ @Override
+ public byte getSerializedTypeTag() {
+ if (isTagged()) {
+ return getByteArray()[getStartOffset()];
+ }
+ return typeTag.serialize();
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ if (isTagged()) {
+ return ATypeTag.VALUE_TYPE_MAPPING[getSerializedTypeTag()];
+ }
+ return typeTag;
+ }
+
+ @Override
+ public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ void init(byte[] data, int offset, int length) {
+ //noOp
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/GenericLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/GenericLazyVisitablePointable.java
new file mode 100644
index 0000000..f6d06ee
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/GenericLazyVisitablePointable.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This is a generic lazy visitable-pointable for tagged values (a.k.a. open values). Each nested visitable-pointable
+ * should only allocate a single instance of this class and reuse it for every open value.
+ */
+public class GenericLazyVisitablePointable extends AbstractLazyVisitablePointable {
+ private RecordLazyVisitablePointable object;
+ private VariableListLazyVisitablePointable array;
+ private VariableListLazyVisitablePointable multiset;
+ private FlatLazyVisitablePointable flat;
+
+ private AbstractLazyVisitablePointable current;
+
+ public GenericLazyVisitablePointable() {
+ super(true);
+ }
+
+ @Override
+ public final byte getSerializedTypeTag() {
+ return current.getSerializedTypeTag();
+ }
+
+ @Override
+ public final ATypeTag getTypeTag() {
+ return current.getTypeTag();
+ }
+
+ @Override
+ public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ switch (current.getTypeTag()) {
+ case OBJECT:
+ return visitor.visit(object, arg);
+ case ARRAY:
+ return visitor.visit(array, arg);
+ case MULTISET:
+ return visitor.visit(multiset, arg);
+ default:
+ return visitor.visit(flat, arg);
+ }
+ }
+
+ @Override
+ void init(byte[] data, int offset, int length) {
+ ATypeTag typeTag = ATypeTag.VALUE_TYPE_MAPPING[data[offset]];
+ AbstractLazyVisitablePointable visitable = getOrCreateVisitablePointable(typeTag);
+ visitable.set(data, offset, length);
+ current = visitable;
+ }
+
+ private AbstractLazyVisitablePointable getOrCreateVisitablePointable(ATypeTag typeTag) {
+ switch (typeTag) {
+ case OBJECT:
+ object = object == null ? new RecordLazyVisitablePointable(true) : object;
+ return object;
+ case ARRAY:
+ array = array == null ? new VariableListLazyVisitablePointable(true,
+ DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE) : array;
+ return array;
+ case MULTISET:
+ multiset = multiset == null ? new VariableListLazyVisitablePointable(true,
+ DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE) : multiset;
+ return multiset;
+ default:
+ flat = flat == null ? new FlatLazyVisitablePointable(true, ATypeTag.ANY) : flat;
+ return flat;
+ }
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/ILazyVisitablePointableVisitor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/ILazyVisitablePointableVisitor.java
new file mode 100644
index 0000000..50116cd
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/ILazyVisitablePointableVisitor.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * A visitor for ADM values which utilizes the lazy visitable:
+ *
+ * @param <R> return type
+ * @param <T> argument type
+ * @see AbstractLazyVisitablePointable
+ * @see AbstractLazyNestedVisitablePointable
+ */
+public interface ILazyVisitablePointableVisitor<R, T> {
+ /**
+ * Visit record value
+ *
+ * @param pointable either typed {@link TypedRecordLazyVisitablePointable} or
+ * untyped {@link RecordLazyVisitablePointable}
+ * @param arg visitor argument
+ * @return return value
+ */
+ R visit(RecordLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+
+ /**
+ * Visit list value
+ *
+ * @param pointable either a list with fixed-length items {@link FixedListLazyVisitablePointable} or
+ * a list with variable-length items {@link VariableListLazyVisitablePointable}
+ * @param arg visitor argument
+ * @return return value
+ */
+
+ R visit(AbstractListLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+
+ /**
+ * Atomic values
+ *
+ * @param pointable any flat item (e.g., {@link org.apache.asterix.om.types.ATypeTag#BIGINT}
+ * @param arg visitor argument
+ * @return return value
+ * @throws HyracksDataException
+ */
+ R visit(FlatLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/RecordLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/RecordLazyVisitablePointable.java
new file mode 100644
index 0000000..154b9f2
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/RecordLazyVisitablePointable.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+/**
+ * This implementation is to handle {@link ATypeTag#OBJECT} with open fields only
+ */
+public class RecordLazyVisitablePointable extends AbstractLazyNestedVisitablePointable {
+ protected final VoidPointable currentFieldName;
+ protected final AbstractLazyVisitablePointable openVisitable;
+ private int openValuesOffset;
+ private int numberOfOpenChildren;
+
+ public RecordLazyVisitablePointable(boolean tagged) {
+ super(tagged, ATypeTag.OBJECT);
+ currentFieldName = new VoidPointable();
+ openVisitable = new GenericLazyVisitablePointable();
+ }
+
+ @Override
+ public void nextChild() throws HyracksDataException {
+ byte[] data = getByteArray();
+
+ //set field name
+ int fieldNameLength = NonTaggedFormatUtil.getFieldValueLength(data, openValuesOffset, ATypeTag.STRING, false);
+ currentFieldName.set(data, openValuesOffset, fieldNameLength);
+ openValuesOffset += fieldNameLength;
+
+ //set Type tag
+ currentChildTypeTag = data[openValuesOffset];
+
+ //set value
+ int valueLength = NonTaggedFormatUtil.getFieldValueLength(data, openValuesOffset, getChildTypeTag(), true) + 1;
+ currentValue.set(data, openValuesOffset, valueLength);
+ openValuesOffset += valueLength;
+ }
+
+ @Override
+ public boolean isTaggedChild() {
+ return true;
+ }
+
+ @Override
+ public int getNumberOfChildren() {
+ return numberOfOpenChildren;
+ }
+
+ public IValueReference getFieldName() {
+ return currentFieldName;
+ }
+
+ @Override
+ public AbstractLazyVisitablePointable getChildVisitablePointable() throws HyracksDataException {
+ openVisitable.set(getChildValue());
+ return openVisitable;
+ }
+
+ @Override
+ public final <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ void init(byte[] data, int offset, int length) {
+ initOpenPart(data, offset);
+ }
+
+ /* ******************************************************
+ * Init Open part
+ * ******************************************************
+ */
+ protected int initOpenPart(byte[] data, int pointer) {
+ //+1 for type tag and +4 for the length
+ int skipTypeTag = isTagged() ? 1 : 0;
+ int currentPointer = pointer + skipTypeTag + 4;
+
+ boolean isExpanded = data[currentPointer] == 1;
+ //Advance to numberOfClosedChildren or
+ currentPointer++;
+
+ if (isExpanded) {
+ /*
+ * -(isTagged() ? 0 : 1) because the open part is computed - in the record builder - with the
+ * assumption that the record type tag is always there. Hence, if the record contains a type tag at the
+ * beginning of the record, we subtract 0 (noop). However, if the record doesn't contain a tag, then we
+ * need to subtract by 1 to get the correct offset of the open part (i.e., as if the tag existed).
+ */
+ int openPartStart =
+ pointer + AInt32SerializerDeserializer.getInt(data, currentPointer) - (isTagged() ? 0 : 1);
+ //Skip open part offset to the beginning of closed part
+ currentPointer += 4;
+ //Number of children in the open part
+ numberOfOpenChildren = AInt32SerializerDeserializer.getInt(data, openPartStart);
+ //Skip the numberOfOpenChildren and the hashOffsetPair to the first open value
+ openValuesOffset = openPartStart + 4 + 8 * numberOfOpenChildren;
+ } else {
+ numberOfOpenChildren = 0;
+ }
+
+ return currentPointer;
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/TypedRecordLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/TypedRecordLazyVisitablePointable.java
new file mode 100644
index 0000000..19eb076
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/TypedRecordLazyVisitablePointable.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+/**
+ * This implementation extends {@link RecordLazyVisitablePointable} to handle {@link ATypeTag#OBJECT} with open and
+ * closed fields
+ */
+public class TypedRecordLazyVisitablePointable extends RecordLazyVisitablePointable {
+ private static final IPointable MISSING_POINTABLE = createConstantPointable(ATypeTag.MISSING);
+ private static final IPointable NULL_POINTABLE = createConstantPointable(ATypeTag.NULL);
+ private final ARecordType recordType;
+
+ //Closed values
+ private final IValueReference[] closedFieldNames;
+ private final AbstractLazyVisitablePointable[] closedVisitables;
+ private final int numberOfClosedChildren;
+ private final ATypeTag[] closedChildTags;
+ //Record builder computes the fields' offset as if the type tag exists
+ private final int actualChildOffset;
+ private int currentIndex;
+ private int closedValuesOffset;
+
+ /**
+ * A constructor for the root record
+ *
+ * @param rootRecordType root record type
+ */
+ public TypedRecordLazyVisitablePointable(ARecordType rootRecordType) {
+ this(true, rootRecordType);
+ }
+
+ public TypedRecordLazyVisitablePointable(boolean tagged, ARecordType recordType) {
+ super(tagged);
+ Objects.requireNonNull(recordType);
+ this.recordType = recordType;
+ numberOfClosedChildren = this.recordType.getFieldTypes().length;
+ closedFieldNames = createSerializedClosedFieldNames(this.recordType);
+ closedVisitables = createClosedVisitables(this.recordType);
+ closedChildTags = createInitialClosedTypeTags(this.recordType);
+ //-1 if not tagged. The offsets were calculated as if the tag exists.
+ actualChildOffset = isTagged() ? 0 : -1;
+ }
+
+ @Override
+ public void nextChild() throws HyracksDataException {
+ currentIndex++;
+ if (isTaggedChild()) {
+ super.nextChild();
+ } else {
+ setClosedValueInfo();
+ }
+ }
+
+ @Override
+ public boolean isTaggedChild() {
+ return currentIndex >= numberOfClosedChildren;
+ }
+
+ @Override
+ public int getNumberOfChildren() {
+ return numberOfClosedChildren + super.getNumberOfChildren();
+ }
+
+ @Override
+ public AbstractLazyVisitablePointable getChildVisitablePointable() throws HyracksDataException {
+ AbstractLazyVisitablePointable visitablePointable;
+ if (isTaggedChild()) {
+ visitablePointable = openVisitable;
+ } else {
+ visitablePointable = closedVisitables[currentIndex];
+ }
+ visitablePointable.set(getChildValue());
+ return visitablePointable;
+ }
+
+ private void setClosedValueInfo() throws HyracksDataException {
+ ATypeTag typeTag = closedChildTags[currentIndex];
+ if (typeTag == ATypeTag.NULL) {
+ currentValue.set(NULL_POINTABLE);
+ } else if (typeTag == ATypeTag.MISSING) {
+ currentValue.set(MISSING_POINTABLE);
+ } else {
+ byte[] data = getByteArray();
+ int offset =
+ getStartOffset() + AInt32SerializerDeserializer.getInt(data, closedValuesOffset + 4 * currentIndex)
+ + actualChildOffset;
+ int length = NonTaggedFormatUtil.getFieldValueLength(data, offset, typeTag, false);
+ currentValue.set(data, offset, length);
+ }
+ currentFieldName.set(closedFieldNames[currentIndex]);
+ currentChildTypeTag = typeTag.serialize();
+ }
+
+ /* ******************************************************
+ * Init Open part
+ * ******************************************************
+ */
+ @Override
+ void init(byte[] data, int offset, int length) {
+ /*
+ * Skip length and the type tag if the current record contains a tag. Only the root can be tagged and typed
+ * at the same time. Nested typed records will not be tagged.
+ */
+ int skipTag = isTagged() ? 1 : 0;
+ currentIndex = -1;
+ //initOpenPart first. It will skip type tag and length.
+ int pointer = recordType.isOpen() ? initOpenPart(data, offset) : offset + skipTag + 4;
+ initClosedPart(pointer, data);
+ }
+
+ private void initClosedPart(int pointer, byte[] data) {
+ //+4 to skip the number of closed children
+ int currentPointer = pointer + 4;
+ if (NonTaggedFormatUtil.hasOptionalField(recordType)) {
+ initClosedChildrenTags(data, currentPointer);
+ currentPointer =
+ (numberOfClosedChildren % 4 == 0 ? numberOfClosedChildren / 4 : numberOfClosedChildren / 4 + 1);
+ }
+ closedValuesOffset = currentPointer;
+ }
+
+ private static IPointable createConstantPointable(ATypeTag tag) {
+ byte[] data = { tag.serialize() };
+ IPointable value = new VoidPointable();
+ value.set(data, 0, 1);
+ return value;
+ }
+
+ private void initClosedChildrenTags(byte[] data, int nullBitMapOffset) {
+ IAType[] types = recordType.getFieldTypes();
+ for (int i = 0; i < numberOfClosedChildren; i++) {
+ byte nullMissingOrValue = data[nullBitMapOffset + i / 4];
+ if (RecordUtil.isNull(nullMissingOrValue, i)) {
+ closedChildTags[i] = ATypeTag.NULL;
+ } else if (RecordUtil.isMissing(nullMissingOrValue, i)) {
+ closedChildTags[i] = ATypeTag.MISSING;
+ } else {
+ IAType type = types[i];
+ type = type.getTypeTag() == ATypeTag.UNION ? ((AUnionType) type).getActualType() : type;
+ closedChildTags[i] = type.getTypeTag();
+ }
+ }
+ }
+
+ private static ATypeTag[] createInitialClosedTypeTags(ARecordType recordType) {
+ IAType[] types = recordType.getFieldTypes();
+ ATypeTag[] typeTags = new ATypeTag[types.length];
+ for (int i = 0; i < types.length; i++) {
+ IAType type = types[i];
+ if (type.getTypeTag() == ATypeTag.UNION) {
+ type = ((AUnionType) type).getActualType();
+ }
+ typeTags[i] = type.getTypeTag();
+ }
+ return typeTags;
+ }
+
+ private static IValueReference[] createSerializedClosedFieldNames(ARecordType recordType) {
+ UTF8StringWriter writer = new UTF8StringWriter();
+ AMutableString mutableString = new AMutableString("");
+ AStringSerializerDeserializer serDer = new AStringSerializerDeserializer(writer, null);
+
+ String[] fieldNames = recordType.getFieldNames();
+ IValueReference[] fieldNameReferences = new IValueReference[fieldNames.length];
+ for (int i = 0; i < fieldNameReferences.length; i++) {
+ mutableString.setValue(fieldNames[i]);
+ fieldNameReferences[i] = createFieldName(mutableString, serDer);
+ }
+ return fieldNameReferences;
+ }
+
+ private static IValueReference createFieldName(AMutableString mutableString, AStringSerializerDeserializer serDer) {
+ ArrayBackedValueStorage storage = new ArrayBackedValueStorage();
+ try {
+ serDer.serialize(mutableString, storage.getDataOutput());
+ } catch (HyracksDataException e) {
+ throw new IllegalStateException(e);
+ }
+ return storage;
+ }
+
+ private static AbstractLazyVisitablePointable[] createClosedVisitables(ARecordType recordType) {
+ IAType[] types = recordType.getFieldTypes();
+ AbstractLazyVisitablePointable[] visitables = new AbstractLazyVisitablePointable[types.length];
+ for (int i = 0; i < types.length; i++) {
+ visitables[i] = createVisitable(types[i]);
+ }
+ return visitables;
+ }
+
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/VariableListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/VariableListLazyVisitablePointable.java
new file mode 100644
index 0000000..9e4ab9f
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/lazy/VariableListLazyVisitablePointable.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.lazy;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This implementation is for {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET} with variable-length items.
+ */
+public class VariableListLazyVisitablePointable extends AbstractListLazyVisitablePointable {
+ private final ATypeTag itemTag;
+ //List builder computes the items' offset as if the type tag exists
+ private final int actualChildOffset;
+
+ public VariableListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+ super(tagged, listType);
+ itemTag = listType.getItemType().getTypeTag();
+ //-1 if not tagged. The offsets were calculated as if the tag exists.
+ actualChildOffset = isTagged() ? 0 : -1;
+ }
+
+ @Override
+ public void nextChild() throws HyracksDataException {
+ byte[] data = getByteArray();
+ int itemOffset = getStartOffset() + AInt32SerializerDeserializer.getInt(data, itemsOffset + currentIndex * 4)
+ + actualChildOffset;
+ ATypeTag itemTypeTag = processTypeTag(data, itemOffset);
+ int itemSize = NonTaggedFormatUtil.getFieldValueLength(data, itemOffset, itemTypeTag, isTaggedChild());
+ currentValue.set(data, itemOffset, itemSize);
+ currentIndex++;
+ }
+
+ private ATypeTag processTypeTag(byte[] data, int itemOffset) {
+ if (itemTag == ATypeTag.ANY) {
+ currentChildTypeTag = data[itemOffset];
+ }
+ return itemTag;
+ }
+
+ @Override
+ public boolean isTaggedChild() {
+ return itemTag == ATypeTag.ANY;
+ }
+
+ @Override
+ AbstractLazyVisitablePointable createVisitablePointable(IAType itemType) {
+ if (itemType.getTypeTag() != ATypeTag.ANY) {
+ return createVisitable(itemType);
+ }
+ return new GenericLazyVisitablePointable();
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AListTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AListTypeComputer.java
index e6e491c..21e88e2 100755
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AListTypeComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AListTypeComputer.java
@@ -41,6 +41,8 @@
public static final AListTypeComputer INSTANCE_INSERT = new AListTypeComputer(3, false, true, false);
public static final AListTypeComputer INSTANCE_REPLACE = new AListTypeComputer(3, false, true, false);
public static final AListTypeComputer INSTANCE_SLICE = new AListTypeComputer(-1, false, false, true);
+ public static final AListTypeComputer INSTANCE_MOVE = new AListTypeComputer(3, false, false, true);
+ public static final AListTypeComputer INSTANCE_SWAP = new AListTypeComputer(3, false, false, true);
private final int minNumArgs;
private final boolean listIsLast;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AbstractRecordFunctionTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AbstractRecordFunctionTypeComputer.java
new file mode 100644
index 0000000..580cab4
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AbstractRecordFunctionTypeComputer.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+/**
+ * Base type computer for the following record-functions:
+ * 1. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_ADD}
+ * 2. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_PUT}
+ * 3. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_REMOVE}
+ * 4. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_RENAME}
+ */
+public abstract class AbstractRecordFunctionTypeComputer implements IResultTypeComputer {
+ @Override
+ public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
+ IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
+ AbstractFunctionCallExpression functionCallExpression = (AbstractFunctionCallExpression) expression;
+
+ // Get our record type.
+ ILogicalExpression arg0 = functionCallExpression.getArguments().get(0).getValue();
+ IAType type0 = (IAType) env.getType(arg0);
+ IAType actualType0 = TypeComputeUtils.getActualType(type0);
+ ATypeTag tag0 = actualType0.getTypeTag();
+ if (tag0 == ATypeTag.ANY) {
+ return wrapTypeWithUnknown(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, true, true);
+ } else if (tag0 == ATypeTag.MISSING) {
+ // Our output is always going to be MISSING.
+ return BuiltinType.AMISSING;
+ } else if (tag0 != ATypeTag.OBJECT) {
+ // Our output is always going to be NULL.
+ return BuiltinType.ANULL;
+ }
+ boolean isOutputMissable = TypeHelper.canBeMissing(type0);
+ boolean isOutputNullable = TypeHelper.canBeNull(type0);
+ ARecordType inputRecordType = TypeComputeUtils.extractRecordType(actualType0);
+
+ // Our second argument should be of type "string".
+ ILogicalExpression arg1 = functionCallExpression.getArguments().get(1).getValue();
+ IAType type1 = (IAType) env.getType(arg1);
+ IAType actualType1 = TypeComputeUtils.getActualType(type1);
+ ATypeTag tag1 = actualType1.getTypeTag();
+ if (tag1 == ATypeTag.ANY) {
+ // We cannot infer the type of our second argument-- our output may be MISSING or NULL.
+ return wrapTypeWithUnknown(type0, true, true);
+ } else if (tag1 == ATypeTag.MISSING) {
+ // Our output is always going to be MISSING.
+ return BuiltinType.AMISSING;
+ } else if (tag1 != ATypeTag.STRING) {
+ // Our output is always going to be NULL.
+ return BuiltinType.ANULL;
+ }
+ isOutputMissable |= TypeHelper.canBeMissing(type1);
+ isOutputNullable |= TypeHelper.canBeNull(type1);
+
+ // Compute our type.
+ return computeTypeImpl(functionCallExpression, env, inputRecordType, isOutputMissable, isOutputNullable);
+ }
+
+ protected abstract IAType computeTypeImpl(AbstractFunctionCallExpression functionCallExpression,
+ IVariableTypeEnvironment env, ARecordType inputRecordType, boolean isOutputMissable,
+ boolean isOutputNullable) throws AlgebricksException;
+
+ protected static IAType wrapTypeWithUnknown(IAType originalType, boolean isMissable, boolean isNullable) {
+ if (isNullable && isMissable) {
+ return AUnionType.createUnknownableType(originalType);
+ } else if (isNullable) { // && !isMissable
+ return AUnionType.createNullableType(originalType);
+ } else if (isMissable) { // && !isNullable
+ return AUnionType.createMissableType(originalType);
+ } else {
+ return originalType;
+ }
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddTypeComputer.java
new file mode 100644
index 0000000..4815340
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddTypeComputer.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.util.Arrays;
+
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public class RecordAddTypeComputer extends AbstractRecordFunctionTypeComputer {
+ public static final RecordAddTypeComputer INSTANCE = new RecordAddTypeComputer();
+
+ private RecordAddTypeComputer() {
+ }
+
+ @Override
+ public IAType computeTypeImpl(AbstractFunctionCallExpression functionCallExpression, IVariableTypeEnvironment env,
+ ARecordType inputRecordType, boolean isOutputMissable, boolean isOutputNullable)
+ throws AlgebricksException {
+ // If our third argument is missing, then just return the type of our first argument.
+ ILogicalExpression arg2 = functionCallExpression.getArguments().get(2).getValue();
+ IAType type2 = (IAType) env.getType(arg2);
+ IAType actualType2 = TypeComputeUtils.getActualType(type2);
+ ATypeTag tag2 = actualType2.getTypeTag();
+ if (tag2 == ATypeTag.MISSING) {
+ IAType type0 = (IAType) env.getType(functionCallExpression.getArguments().get(0).getValue());
+ return wrapTypeWithUnknown(type0, isOutputMissable, isOutputNullable);
+ }
+
+ // We expect a constant for our second argument.
+ ILogicalExpression arg1 = functionCallExpression.getArguments().get(1).getValue();
+ if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return wrapTypeWithUnknown(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, isOutputMissable,
+ isOutputNullable);
+ }
+ ConstantExpression constantExpression = (ConstantExpression) arg1;
+ AsterixConstantValue constantValue = (AsterixConstantValue) constantExpression.getValue();
+ String newFieldName = ((AString) constantValue.getObject()).getStringValue();
+
+ // If our field is found, return the original record type. Otherwise, add our new field.
+ ARecordType outputRecordType = inputRecordType;
+ if (!Arrays.asList(inputRecordType.getFieldNames()).contains(newFieldName)) {
+ String[] fieldNames = new String[inputRecordType.getFieldNames().length + 1];
+ IAType[] fieldTypes = new IAType[inputRecordType.getFieldTypes().length + 1];
+ int currentCursorPosition = 0;
+ for (; currentCursorPosition < inputRecordType.getFieldNames().length; currentCursorPosition++) {
+ fieldNames[currentCursorPosition] = inputRecordType.getFieldNames()[currentCursorPosition];
+ fieldTypes[currentCursorPosition] = inputRecordType.getFieldTypes()[currentCursorPosition];
+ }
+ fieldNames[currentCursorPosition] = newFieldName;
+ fieldTypes[currentCursorPosition] = type2;
+ String inputTypeName = inputRecordType.getTypeName();
+ String outputTypeName = inputTypeName != null ? inputTypeName + "_add_" + newFieldName : null;
+ outputRecordType = new ARecordType(outputTypeName, fieldNames, fieldTypes, inputRecordType.isOpen());
+ }
+ return wrapTypeWithUnknown(outputRecordType, isOutputMissable, isOutputNullable);
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordPutTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordPutTypeComputer.java
new file mode 100644
index 0000000..a92907d
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordPutTypeComputer.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public class RecordPutTypeComputer extends AbstractRecordFunctionTypeComputer {
+ public static final RecordPutTypeComputer INSTANCE = new RecordPutTypeComputer();
+
+ private RecordPutTypeComputer() {
+ }
+
+ @Override
+ public IAType computeTypeImpl(AbstractFunctionCallExpression functionCallExpression, IVariableTypeEnvironment env,
+ ARecordType inputRecordType, boolean isOutputMissable, boolean isOutputNullable)
+ throws AlgebricksException {
+ // Extract the type of our third argument. If it is MISSING, then we are performing a field removal.
+ ILogicalExpression arg2 = functionCallExpression.getArguments().get(2).getValue();
+ IAType type2 = (IAType) env.getType(arg2);
+ IAType actualType2 = TypeComputeUtils.getActualType(type2);
+ boolean isFieldRemoval = actualType2.getTypeTag() == ATypeTag.MISSING;
+
+ // We expect a constant for our second argument.
+ ILogicalExpression arg1 = functionCallExpression.getArguments().get(1).getValue();
+ if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return wrapTypeWithUnknown(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, isOutputMissable,
+ isOutputNullable);
+ }
+ ConstantExpression constantExpression = (ConstantExpression) arg1;
+ AsterixConstantValue constantValue = (AsterixConstantValue) constantExpression.getValue();
+ String newFieldName = ((AString) constantValue.getObject()).getStringValue();
+
+ // Remove or replace our field name and type (dependent on the type of our third argument).
+ boolean fieldFound = false;
+ List<String> outputFieldNames = new ArrayList<>();
+ List<IAType> outputFieldTypes = new ArrayList<>();
+ for (int i = 0; i < inputRecordType.getFieldNames().length; i++) {
+ String inputFieldName = inputRecordType.getFieldNames()[i];
+ IAType inputFieldType = inputRecordType.getFieldTypes()[i];
+ if (!inputFieldName.equals(newFieldName)) {
+ outputFieldNames.add(inputFieldName);
+ outputFieldTypes.add(inputFieldType);
+
+ } else {
+ fieldFound = true;
+ if (!isFieldRemoval) {
+ // Replace our input field type.
+ outputFieldNames.add(inputFieldName);
+ outputFieldTypes.add(type2);
+ }
+ }
+ }
+
+ // Build our output record type.
+ ARecordType outputRecordType;
+ String inputTypeName = inputRecordType.getTypeName();
+ boolean doesRecordHaveTypeName = inputTypeName != null;
+ if (fieldFound && isFieldRemoval) {
+ // We have removed our argument field.
+ String outputTypeName = doesRecordHaveTypeName ? inputTypeName + "_remove_" + newFieldName : null;
+ outputRecordType = new ARecordType(outputTypeName, outputFieldNames.toArray(String[]::new),
+ outputFieldTypes.toArray(IAType[]::new), inputRecordType.isOpen());
+ } else if (fieldFound) { // && !isFieldRemoval
+ // We have replaced our argument field.
+ String outputTypeName = doesRecordHaveTypeName ? inputTypeName + "_replaced_" + newFieldName : null;
+ outputRecordType = new ARecordType(outputTypeName, outputFieldNames.toArray(String[]::new),
+ outputFieldTypes.toArray(IAType[]::new), inputRecordType.isOpen());
+ } else if (!isFieldRemoval) { // && !wasFieldFound
+ // We need to insert our argument field.
+ outputFieldNames.add(newFieldName);
+ outputFieldTypes.add(type2);
+ String outputTypeName = doesRecordHaveTypeName ? inputTypeName + "_add_" + newFieldName : null;
+ outputRecordType = new ARecordType(outputTypeName, outputFieldNames.toArray(String[]::new),
+ outputFieldTypes.toArray(IAType[]::new), inputRecordType.isOpen());
+ } else { // isFieldRemoval && !wasFieldFound
+ // We have not found the field to remove.
+ outputRecordType = inputRecordType;
+ }
+ return wrapTypeWithUnknown(outputRecordType, isOutputMissable, isOutputNullable);
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
index 83af00d..bdb7277 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
@@ -42,6 +42,7 @@
import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeHelper;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -84,6 +85,7 @@
case STRING:
String fn = ((AString) fieldName).getStringValue();
fieldNameSet.add(fn);
+ pathList.add(List.of(fn));
break;
case ARRAY:
AOrderedList pathOrdereList = (AOrderedList) fieldName;
@@ -237,16 +239,18 @@
IAType[] fieldTypes = inputRecordType.getFieldTypes();
for (int i = 0; i < fieldNames.length; i++) {
+ IAType originalType = fieldTypes[i];
+ IAType actualType = TypeComputeUtils.getActualType(originalType);
if (!fieldNameSet.contains(fieldNames[i])) { // The main field is to be kept
addField(inputRecordType, fieldNames[i], resultFieldNames, resultFieldTypes);
- } else if (!pathList.isEmpty() && fieldTypes[i].getTypeTag() == ATypeTag.OBJECT) {
- ARecordType subRecord = (ARecordType) fieldTypes[i];
+ } else if (!pathList.isEmpty() && actualType.getTypeTag() == ATypeTag.OBJECT) {
+ ARecordType subRecord = (ARecordType) actualType;
fieldPathStack.push(fieldNames[i]);
- subRecord = deepCheckAndCopy(fieldPathStack, subRecord, pathList, inputRecordType.isOpen());
+ subRecord = deepCheckAndCopy(fieldPathStack, subRecord, pathList, subRecord.isOpen());
fieldPathStack.pop();
if (subRecord != null) {
resultFieldNames.add(fieldNames[i]);
- resultFieldTypes.add(subRecord);
+ resultFieldTypes.add(wrapWithOriginalType(subRecord, originalType));
}
}
}
@@ -259,6 +263,18 @@
}
+ private static IAType wrapWithOriginalType(IAType typeToModify, IAType originalType) {
+ if (TypeHelper.canBeMissing(originalType) && !TypeHelper.canBeNull(originalType)) {
+ return AUnionType.createMissableType(typeToModify);
+ } else if (!TypeHelper.canBeMissing(originalType) && TypeHelper.canBeNull(originalType)) {
+ return AUnionType.createNullableType(typeToModify);
+ } else if (TypeHelper.canBeUnknown(originalType)) {
+ return AUnionType.createUnknownableType(typeToModify);
+ } else {
+ return typeToModify;
+ }
+ }
+
/**
* Comparison elements of two paths
* Note: l2 uses a LIFO insert and removal.
@@ -314,12 +330,14 @@
for (int i = 0; i < srcFieldNames.length; i++) {
fieldPath.push(srcFieldNames[i]);
if (!isRemovePath(fieldPath, pathList)) {
- if (srcFieldTypes[i].getTypeTag() == ATypeTag.OBJECT) {
- ARecordType subRecord = (ARecordType) srcFieldTypes[i];
+ IAType originalType = srcFieldTypes[i];
+ IAType actualType = TypeComputeUtils.getActualType(originalType);
+ if (actualType.getTypeTag() == ATypeTag.OBJECT) {
+ ARecordType subRecord = (ARecordType) actualType;
subRecord = deepCheckAndCopy(fieldPath, subRecord, pathList, isOpen);
if (subRecord != null) {
destFieldNames.add(srcFieldNames[i]);
- destFieldTypes.add(subRecord);
+ destFieldTypes.add(wrapWithOriginalType(subRecord, originalType));
}
} else {
destFieldNames.add(srcFieldNames[i]);
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveTypeComputer.java
new file mode 100644
index 0000000..c447e65
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveTypeComputer.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.util.Arrays;
+
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public class RecordRemoveTypeComputer extends AbstractRecordFunctionTypeComputer {
+ public static final RecordRemoveTypeComputer INSTANCE = new RecordRemoveTypeComputer();
+
+ private RecordRemoveTypeComputer() {
+ }
+
+ @Override
+ public IAType computeTypeImpl(AbstractFunctionCallExpression functionCallExpression, IVariableTypeEnvironment env,
+ ARecordType inputRecordType, boolean isOutputMissable, boolean isOutputNullable) {
+ // We expect a CONSTANT expression. Otherwise, defer the removal to runtime.
+ ILogicalExpression arg1 = functionCallExpression.getArguments().get(1).getValue();
+ if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return wrapTypeWithUnknown(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, isOutputMissable,
+ isOutputNullable);
+ }
+ ConstantExpression constantExpression = (ConstantExpression) arg1;
+ AsterixConstantValue constantValue = (AsterixConstantValue) constantExpression.getValue();
+ String fieldName = ((AString) constantValue.getObject()).getStringValue();
+
+ // If our field is found, remove it. Otherwise, return the original record type.
+ ARecordType outputRecordType = inputRecordType;
+ if (Arrays.asList(inputRecordType.getFieldNames()).contains(fieldName)) {
+ String[] fieldNames = new String[inputRecordType.getFieldNames().length - 1];
+ IAType[] fieldTypes = new IAType[inputRecordType.getFieldTypes().length - 1];
+ int currentOutputCursor = 0;
+ for (int i = 0; i < inputRecordType.getFieldNames().length; i++) {
+ String inputName = inputRecordType.getFieldNames()[i];
+ IAType inputType = inputRecordType.getFieldTypes()[i];
+ if (!inputName.equals(fieldName)) {
+ fieldNames[currentOutputCursor] = inputName;
+ fieldTypes[currentOutputCursor] = inputType;
+ currentOutputCursor++;
+ }
+ }
+ String inputTypeName = inputRecordType.getTypeName();
+ String outputTypeName = inputTypeName != null ? inputTypeName + "_remove_" + fieldName : null;
+ outputRecordType = new ARecordType(outputTypeName, fieldNames, fieldTypes, inputRecordType.isOpen());
+ }
+ return wrapTypeWithUnknown(outputRecordType, isOutputMissable, isOutputNullable);
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRenameTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRenameTypeComputer.java
new file mode 100644
index 0000000..4302082
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRenameTypeComputer.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.util.Arrays;
+
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeHelper;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public class RecordRenameTypeComputer extends AbstractRecordFunctionTypeComputer {
+ public static final RecordRenameTypeComputer INSTANCE = new RecordRenameTypeComputer();
+
+ private RecordRenameTypeComputer() {
+ }
+
+ @Override
+ public IAType computeTypeImpl(AbstractFunctionCallExpression functionCallExpression, IVariableTypeEnvironment env,
+ ARecordType inputRecordType, boolean isOutputMissable, boolean isOutputNullable)
+ throws AlgebricksException {
+ // Our third argument should be of type "string".
+ ILogicalExpression arg2 = functionCallExpression.getArguments().get(2).getValue();
+ IAType type2 = (IAType) env.getType(arg2);
+ IAType actualType2 = TypeComputeUtils.getActualType(type2);
+ ATypeTag tag2 = actualType2.getTypeTag();
+ if (tag2 == ATypeTag.ANY) {
+ // We cannot infer the type of our third argument-- our output may be MISSING or NULL.
+ return AUnionType.createUnknownableType(inputRecordType, inputRecordType.getTypeName() + "?");
+ } else if (tag2 == ATypeTag.MISSING) {
+ // Our output is always going to be MISSING.
+ return BuiltinType.AMISSING;
+ } else if (tag2 != ATypeTag.STRING) {
+ // Our output is always going to be NULL.
+ return BuiltinType.ANULL;
+ }
+ isOutputMissable |= TypeHelper.canBeMissing(type2);
+ isOutputNullable |= TypeHelper.canBeNull(type2);
+
+ // We expect a CONSTANT expression for both arguments. Otherwise, defer the replacement to runtime.
+ ILogicalExpression arg1 = functionCallExpression.getArguments().get(1).getValue();
+ if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT
+ || arg2.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return wrapTypeWithUnknown(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, isOutputMissable,
+ isOutputNullable);
+ }
+ ConstantExpression arg1ConstantExpression = (ConstantExpression) arg1;
+ ConstantExpression arg2ConstantExpression = (ConstantExpression) arg2;
+ AsterixConstantValue arg1ConstantValue = (AsterixConstantValue) arg1ConstantExpression.getValue();
+ AsterixConstantValue arg2ConstantValue = (AsterixConstantValue) arg2ConstantExpression.getValue();
+ String oldFieldName = ((AString) arg1ConstantValue.getObject()).getStringValue();
+ String newFieldName = ((AString) arg2ConstantValue.getObject()).getStringValue();
+
+ // If our field is found, replace it. Otherwise, return the original record type.
+ Mutable<Boolean> fieldFound = new MutableObject<>(false);
+ String[] newFieldNames = Arrays.stream(inputRecordType.getFieldNames()).map(f -> {
+ if (f.equals(oldFieldName)) {
+ fieldFound.setValue(true);
+ return newFieldName;
+ } else {
+ return f;
+ }
+ }).toArray(String[]::new);
+ ARecordType outputRecordType;
+ if (!fieldFound.getValue()) {
+ outputRecordType = inputRecordType;
+ } else {
+ String inputTypeName = inputRecordType.getTypeName();
+ String outputTypeName = inputTypeName != null ? inputTypeName + "_replaced_" + oldFieldName : null;
+ outputRecordType = new ARecordType(outputTypeName, newFieldNames, inputRecordType.getFieldTypes(),
+ inputRecordType.isOpen());
+ }
+ return wrapTypeWithUnknown(outputRecordType, isOutputMissable, isOutputNullable);
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
index e256e1b..fd51433 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
@@ -25,7 +25,6 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import org.apache.asterix.common.annotations.IRecordTypeAnnotation;
@@ -73,30 +72,21 @@
private final Set<String> allPossibleAdditionalFieldNames;
/**
- * @param typeName
- * the name of the type
- * @param fieldNames
- * the names of the closed fields
- * @param fieldTypes
- * the types of the closed fields
- * @param isOpen
- * whether the record is open
+ * @param typeName the name of the type
+ * @param fieldNames the names of the closed fields
+ * @param fieldTypes the types of the closed fields
+ * @param isOpen whether the record is open
*/
public ARecordType(String typeName, String[] fieldNames, IAType[] fieldTypes, boolean isOpen) {
this(typeName, fieldNames, fieldTypes, isOpen, null);
}
/**
- * @param typeName
- * the name of the type
- * @param fieldNames
- * the names of the closed fields
- * @param fieldTypes
- * the types of the closed fields
- * @param isOpen
- * whether the record is open
- * @param allPossibleAdditionalFieldNames,
- * all possible additional field names.
+ * @param typeName the name of the type
+ * @param fieldNames the names of the closed fields
+ * @param fieldTypes the types of the closed fields
+ * @param isOpen whether the record is open
+ * @param allPossibleAdditionalFieldNames, all possible additional field names.
*/
public ARecordType(String typeName, String[] fieldNames, IAType[] fieldTypes, boolean isOpen,
Set<String> allPossibleAdditionalFieldNames) {
@@ -189,8 +179,7 @@
/**
* Returns the position of the field in the closed schema or -1 if the field does not exist.
*
- * @param fieldName
- * the name of the field whose position is sought
+ * @param fieldName the name of the field whose position is sought
* @return the position of the field in the closed schema or -1 if the field does not exist.
*/
public int getFieldIndex(String fieldName) {
@@ -206,10 +195,8 @@
}
/**
- * @param subFieldName
- * The full pathname of the child
- * @param parent
- * The type of the parent
+ * @param subFieldName The full pathname of the child
+ * @param parent The type of the parent
* @return the type of the child
*/
@@ -222,8 +209,7 @@
}
/**
- * @param subFieldName
- * The full pathname of the child
+ * @param subFieldName The full pathname of the child
* @return the type of the child
* @throws AsterixException
*/
@@ -248,9 +234,7 @@
}
/**
- *
- * @param subFieldName
- * The full pathname of the field
+ * @param subFieldName The full pathname of the field
* @return The nullability of the field
* @throws AlgebricksException
*/
@@ -283,10 +267,9 @@
/**
* Returns the field type of the field name if it exists, otherwise null.
*
- * @param fieldName
- * the fieldName whose type is sought
+ * @param fieldName the fieldName whose type is sought
* @return the field type of the field name if it exists, otherwise null
- * NOTE: this method doesn't work for nested fields
+ * NOTE: this method doesn't work for nested fields
*/
public IAType getFieldType(String fieldName) {
int fieldPos = getFieldIndex(fieldName);
@@ -299,8 +282,7 @@
/**
* Returns true or false indicating whether or not a field is closed.
*
- * @param fieldName
- * the name of the field to check
+ * @param fieldName the name of the field to check
* @return true if fieldName is a closed field, otherwise false
*/
public boolean isClosedField(String fieldName) {
@@ -360,8 +342,7 @@
}
ARecordType rt = (ARecordType) obj;
return (isOpen == rt.isOpen) && Arrays.deepEquals(fieldNames, rt.fieldNames)
- && Arrays.deepEquals(fieldTypes, rt.fieldTypes)
- && Objects.equals(allPossibleAdditionalFieldNames, rt.allPossibleAdditionalFieldNames);
+ && Arrays.deepEquals(fieldTypes, rt.fieldTypes);
}
@Override
@@ -382,11 +363,7 @@
ObjectNode type = om.createObjectNode();
type.put("type", ARecordType.class.getName());
type.put("name", typeName);
- if (isOpen) {
- type.put("open", true);
- } else {
- type.put("open", false);
- }
+ type.put("open", isOpen);
ArrayNode fields = om.createArrayNode();
for (int i = 0; i < fieldNames.length; i++) {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/visitor/PathStringBuilderForIATypeVisitor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/visitor/PathStringBuilderForIATypeVisitor.java
new file mode 100644
index 0000000..ddc2437
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/visitor/PathStringBuilderForIATypeVisitor.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.types.visitor;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
+
+/**
+ * Produces a path from a type
+ */
+public class PathStringBuilderForIATypeVisitor implements IATypeVisitor<Void, StringBuilder> {
+ private boolean root = true;
+
+ @Override
+ public Void visit(ARecordType recordType, StringBuilder arg) {
+ // here we assume the record type has only one child
+ if (root) {
+ arg.append("$$root");
+ root = false;
+ }
+ arg.append('.');
+ arg.append(recordType.getFieldNames()[0]);
+ recordType.getFieldTypes()[0].accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(AbstractCollectionType collectionType, StringBuilder arg) {
+ arg.append("[*]");
+ collectionType.getItemType().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(AUnionType unionType, StringBuilder arg) {
+ throw new NotImplementedException("Check how to represent this");
+ }
+
+ @Override
+ public Void visitFlat(IAType flatType, StringBuilder arg) {
+ return null;
+ }
+}
diff --git a/asterixdb/asterix-docker/docker/.gitattributes b/asterixdb/asterix-podman/docker/.gitattributes
similarity index 100%
rename from asterixdb/asterix-docker/docker/.gitattributes
rename to asterixdb/asterix-podman/docker/.gitattributes
diff --git a/asterixdb/asterix-docker/docker/Dockerfile b/asterixdb/asterix-podman/docker/Dockerfile
similarity index 100%
rename from asterixdb/asterix-docker/docker/Dockerfile
rename to asterixdb/asterix-podman/docker/Dockerfile
diff --git a/asterixdb/asterix-docker/docker/asterix-configuration.xml b/asterixdb/asterix-podman/docker/asterix-configuration.xml
similarity index 100%
rename from asterixdb/asterix-docker/docker/asterix-configuration.xml
rename to asterixdb/asterix-podman/docker/asterix-configuration.xml
diff --git a/asterixdb/asterix-docker/docker/fbm.adm b/asterixdb/asterix-podman/docker/fbm.adm
similarity index 100%
rename from asterixdb/asterix-docker/docker/fbm.adm
rename to asterixdb/asterix-podman/docker/fbm.adm
diff --git a/asterixdb/asterix-docker/docker/fbu.adm b/asterixdb/asterix-podman/docker/fbu.adm
similarity index 100%
rename from asterixdb/asterix-docker/docker/fbu.adm
rename to asterixdb/asterix-podman/docker/fbu.adm
diff --git a/asterixdb/asterix-docker/docker/supervisord.conf b/asterixdb/asterix-podman/docker/supervisord.conf
similarity index 100%
rename from asterixdb/asterix-docker/docker/supervisord.conf
rename to asterixdb/asterix-podman/docker/supervisord.conf
diff --git a/asterixdb/asterix-docker/docker/twm.adm b/asterixdb/asterix-podman/docker/twm.adm
similarity index 100%
rename from asterixdb/asterix-docker/docker/twm.adm
rename to asterixdb/asterix-podman/docker/twm.adm
diff --git a/asterixdb/asterix-docker/docker/twu.adm b/asterixdb/asterix-podman/docker/twu.adm
similarity index 100%
rename from asterixdb/asterix-docker/docker/twu.adm
rename to asterixdb/asterix-podman/docker/twu.adm
diff --git a/asterixdb/asterix-podman/pom.xml b/asterixdb/asterix-podman/pom.xml
new file mode 100644
index 0000000..23928aa
--- /dev/null
+++ b/asterixdb/asterix-podman/pom.xml
@@ -0,0 +1,156 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-asterixdb</artifactId>
+ <groupId>org.apache.asterix</groupId>
+ <version>0.9.9-SNAPSHOT</version>
+ </parent>
+ <artifactId>asterix-podman</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>asterix-server</artifactId>
+ <version>${project.version}</version>
+ <type>deb</type>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-app</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>1.17.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <root.dir>${basedir}/..</root.dir>
+ </properties>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/setup.sh</exclude>
+ <exclude>src/test/resources/passwd</exclude>
+ <exclude>src/test/resources/socktest/Containerfile</exclude>
+ <exclude>src/test/resources/testenv.conf</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <id>podman.tests</id>
+ <properties>
+ <test.excludes>**/*.java</test.excludes>
+ <itest.includes>**/PodmanPythonFunctionIT.java</itest.includes>
+ <failIfNoTests>false</failIfNoTests>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>nl.lexemmens</groupId>
+ <artifactId>podman-maven-plugin</artifactId>
+ <version>1.8.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>generate-test-resources</phase>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAuth>true</skipAuth>
+ <images>
+ <image>
+ <name>asterixdb/socktest</name>
+ <build>
+ <pull>false</pull>
+ <createLatestTag>true</createLatestTag>
+ <containerFileDir>src/test/resources/socktest</containerFileDir>
+ </build>
+ </image>
+ </images>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-external-data-resources</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+ <directory>../asterix-server/target</directory>
+ <includes>
+ <include>asterix-server*.deb</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
diff --git a/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanPythonFunctionIT.java b/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanPythonFunctionIT.java
new file mode 100644
index 0000000..f0f89cd
--- /dev/null
+++ b/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanPythonFunctionIT.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.podman;
+
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.test.runtime.ExecutionTestUtil;
+import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import com.github.dockerjava.api.DockerClient;
+
+/**
+ * Runs the Python UDF tests within a container using domain sockets.
+ */
+@RunWith(Parameterized.class)
+public class PodmanPythonFunctionIT {
+ public static final DockerImageName ASTERIX_IMAGE = DockerImageName.parse("asterixdb/socktest");
+ @ClassRule
+ public static GenericContainer<?> asterix = new GenericContainer(ASTERIX_IMAGE).withExposedPorts(19004, 5006, 19002)
+ .withFileSystemBind("../asterix-app/", "/var/tmp/asterix-app/", BindMode.READ_WRITE);
+ protected static final String TEST_CONFIG_FILE_NAME = "../asterix-app/src/test/resources/cc.conf";
+ private static final boolean cleanupOnStop = true;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ final TestExecutor testExecutor = new TestExecutor(
+ List.of(InetSocketAddress.createUnresolved(asterix.getHost(), asterix.getMappedPort(19002))));
+ asterix.execInContainer("/opt/setup.sh");
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor, false, true, new PodmanUDFLibrarian(asterix));
+ setEndpoints(testExecutor);
+ testExecutor.waitForClusterActive(60, TimeUnit.SECONDS);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ try {
+ } finally {
+ ExecutionTestUtil.tearDown(cleanupOnStop);
+ DockerClient dc = DockerClientFactory.instance().client();
+ dc.removeImageCmd(ASTERIX_IMAGE.asCanonicalNameString()).withForce(true).exec();
+ }
+ }
+
+ @Parameters(name = "PodmanPythonFunctionIT {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_sqlpp.xml", "testsuite_it_python.xml",
+ "../asterix-app/src/test/resources/runtimets");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public PodmanPythonFunctionIT(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ LangExecutionUtil.test(tcCtx);
+ }
+
+ private static void setEndpoints(TestExecutor testExecutor) {
+ final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
+ final String ip = asterix.getHost();
+ final String nodeId = "asterix_nc";
+ int apiPort = asterix.getMappedPort(19004);
+ ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip, apiPort));
+ testExecutor.setNcEndPoints(ncEndPoints);
+ }
+}
diff --git a/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanUDFLibrarian.java b/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanUDFLibrarian.java
new file mode 100644
index 0000000..025f607
--- /dev/null
+++ b/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanUDFLibrarian.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.podman;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.asterix.app.external.IExternalUDFLibrarian;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.GenericContainer;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.json.JsonReadFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class PodmanUDFLibrarian implements IExternalUDFLibrarian {
+ final GenericContainer<?> asterix;
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public PodmanUDFLibrarian(GenericContainer asterix) {
+ OBJECT_MAPPER.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true);
+ this.asterix = asterix;
+ }
+
+ @Override
+ public void install(URI path, String type, String libPath, Pair<String, String> credentials) throws Exception {
+ Container.ExecResult curlResult = null;
+ int retryCt = 0;
+ while (retryCt < 10) {
+ try {
+ curlResult = asterix.execInContainer("curl", "--no-progress-meter", "-X", "POST", "-u",
+ credentials.first + ":" + credentials.second, "-F",
+ "data=@" + "/var/tmp/asterix-app/" + libPath, "-F", "type=" + type,
+ "http://localhost:19004" + path.getRawPath());
+ handleResponse(curlResult);
+ return;
+ } catch (RuntimeException e) {
+ retryCt++;
+ if (retryCt > 9)
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public void uninstall(URI path, Pair<String, String> credentials) throws IOException, AsterixException {
+ try {
+ Container.ExecResult curlResult = asterix.execInContainer("curl", "-X", "DELETE", "-u",
+ credentials.first + ":" + credentials.second, "http://localhost:19004" + path.getPath());
+ handleResponse(curlResult);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void handleResponse(Container.ExecResult result) throws AsterixException, JsonProcessingException {
+ if (result.getExitCode() != 0) {
+ throw new AsterixException(result.getStderr());
+ }
+ JsonNode resp = OBJECT_MAPPER.readTree(result.getStdout().replace('\0', ' '));
+ if (resp.has("error")) {
+ throw new AsterixException(resp.get("error").toString());
+ }
+ return;
+ }
+}
diff --git a/asterixdb/asterix-podman/src/test/resources/cc.conf b/asterixdb/asterix-podman/src/test/resources/cc.conf
new file mode 100644
index 0000000..e4cbd73
--- /dev/null
+++ b/asterixdb/asterix-podman/src/test/resources/cc.conf
@@ -0,0 +1,36 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=/opt/apache-asterixdb/data/txnlog
+core.dump.dir=/opt/apache-asterixdb/logs/coredump
+iodevices=/opt/apache-asterixdb/data/
+nc.api.port=19004
+
+[nc]
+address=127.0.0.1
+command=asterixnc
+credential.file=/opt/apache-asterixdb/etc/passwd
+jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006
+python.ds.path = /tmp/pyudf.socket
+
+[cc]
+address = 127.0.0.1
+
+[common]
+log.level = INFO
+log.dir = /opt/apache-asterixdb/logs/
diff --git a/asterixdb/asterix-podman/src/test/resources/passwd b/asterixdb/asterix-podman/src/test/resources/passwd
new file mode 100644
index 0000000..a1ea5b0
--- /dev/null
+++ b/asterixdb/asterix-podman/src/test/resources/passwd
@@ -0,0 +1 @@
+admin:$2a$12$JxgDzf/uOn1NS2Y3exhrDOf7JY/eUHQH7HeH90s5Ye2gALoO0FsQy
diff --git a/asterixdb/asterix-podman/src/test/resources/setup.sh b/asterixdb/asterix-podman/src/test/resources/setup.sh
new file mode 100644
index 0000000..e3523aa
--- /dev/null
+++ b/asterixdb/asterix-podman/src/test/resources/setup.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+cd /var/tmp/asterix-app/
+shiv -o target/TweetSent.pyz --site-packages src/test/resources/TweetSent scikit-learn
+cp -a /var/tmp/asterix-app/data/classifications /opt/apache-asterixdb/data/
+cp -a /var/tmp/asterix-app/data/twitter /opt/apache-asterixdb/data/
+cp -a /var/tmp/asterix-app/data/big-object /opt/apache-asterixdb/data/
+mkdir -p /opt/apache-asterixdb/target/data/
+cp -a /var/tmp/asterix-app/target/data/big-object /opt/apache-asterixdb/target/data/
\ No newline at end of file
diff --git a/asterixdb/asterix-podman/src/test/resources/socktest/Containerfile b/asterixdb/asterix-podman/src/test/resources/socktest/Containerfile
new file mode 100644
index 0000000..a7546d5
--- /dev/null
+++ b/asterixdb/asterix-podman/src/test/resources/socktest/Containerfile
@@ -0,0 +1,17 @@
+FROM ubuntu:22.04
+RUN apt -y update
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt -y install systemd openjdk-17-jre-headless unzip wget curl python3-pip python3-venv python3-systemd
+RUN pip3 install shiv msgpack
+COPY target/asterix-server_*all.deb .
+RUN dpkg -i asterix-server*.deb
+COPY src/test/resources/cc.conf /opt/apache-asterixdb/cc.conf
+COPY src/test/resources/passwd /opt/apache-asterixdb/etc/passwd
+RUN mkdir -p /etc/systemd/system/pyudf@.service.d/
+COPY src/test/resources/testenv.conf /etc/systemd/system/pyudf@.service.d/
+COPY src/test/resources/setup.sh /opt
+RUN chmod +x /opt/setup.sh
+RUN systemctl enable asterix-nc asterix-cc pyudf.socket
+
+EXPOSE 19001 19002 19004
+
+CMD [ "/lib/systemd/systemd" ]
diff --git a/asterixdb/asterix-podman/src/test/resources/testenv.conf b/asterixdb/asterix-podman/src/test/resources/testenv.conf
new file mode 100644
index 0000000..0c2f182
--- /dev/null
+++ b/asterixdb/asterix-podman/src/test/resources/testenv.conf
@@ -0,0 +1,3 @@
+[Service]
+Environment="FOO=BAR=BAZ"
+Environment="BAR=BAZ"
diff --git a/asterixdb/asterix-replication/pom.xml b/asterixdb/asterix-replication/pom.xml
index 7e9b4f8..e82a17c 100644
--- a/asterixdb/asterix-replication/pom.xml
+++ b/asterixdb/asterix-replication/pom.xml
@@ -3,7 +3,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-replication</artifactId>
<licenses>
diff --git a/asterixdb/asterix-runtime/pom.xml b/asterixdb/asterix-runtime/pom.xml
index 22958ac..4f2f32a 100644
--- a/asterixdb/asterix-runtime/pom.xml
+++ b/asterixdb/asterix-runtime/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-runtime</artifactId>
<properties>
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractArrayMoveSwapEval.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractArrayMoveSwapEval.java
new file mode 100644
index 0000000..218e5ac
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractArrayMoveSwapEval.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
+
+import java.io.IOException;
+
+import org.apache.asterix.builders.IAsterixListBuilder;
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeTagUtil;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.evaluators.common.ListAccessor;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+/**
+ * Abstract class for the ARRAY_MOVE and ARRAY_SWAP functions as they share a large portion of their code
+ * with each other. Any classes that use this abstract one should override the `buildList` method, as it just
+ * otherwise builds the same input list.
+ */
+public abstract class AbstractArrayMoveSwapEval implements IScalarEvaluator {
+
+ private final ArrayBackedValueStorage storage;
+ private final IScalarEvaluator listArgEval;
+ private final IScalarEvaluator oldIndexEval;
+ private final IScalarEvaluator newIndexEval;
+ private final IPointable listArg;
+ private final IPointable oldIndex;
+ private final IPointable newIndex;
+ private final ListAccessor listAccessor;
+ private IAsterixListBuilder listBuilder;
+ private String funcIdentifier;
+ private IAType inputListType;
+
+ AbstractArrayMoveSwapEval(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx, String funcIdentifier,
+ IAType inputListType) throws HyracksDataException {
+
+ this.funcIdentifier = funcIdentifier;
+ this.inputListType = inputListType;
+ storage = new ArrayBackedValueStorage();
+ listArgEval = args[0].createScalarEvaluator(ctx);
+ oldIndexEval = args[1].createScalarEvaluator(ctx);
+ newIndexEval = args[2].createScalarEvaluator(ctx);
+ listArg = new VoidPointable();
+ oldIndex = new VoidPointable();
+ newIndex = new VoidPointable();
+ listAccessor = new ListAccessor();
+ listBuilder = new OrderedListBuilder();
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+
+ storage.reset();
+
+ // Check that our args aren't missing/null
+ listArgEval.evaluate(tuple, listArg);
+ oldIndexEval.evaluate(tuple, oldIndex);
+ newIndexEval.evaluate(tuple, newIndex);
+ if (PointableHelper.checkAndSetMissingOrNull(result, listArg, oldIndex, newIndex)) {
+ return;
+ }
+
+ byte[] listBytes = listArg.getByteArray();
+ int offset = listArg.getStartOffset();
+ ATypeTag listType = ATYPETAGDESERIALIZER.deserialize(listBytes[offset]);
+
+ byte[] oldIndexBytes = oldIndex.getByteArray();
+ int oldIndexOffset = oldIndex.getStartOffset();
+ ATypeTag oldIndexType = ATYPETAGDESERIALIZER.deserialize((oldIndexBytes[oldIndexOffset]));
+
+ byte[] newIndexBytes = newIndex.getByteArray();
+ int newIndexOffset = newIndex.getStartOffset();
+ ATypeTag newIndexType = ATYPETAGDESERIALIZER.deserialize(newIndexBytes[newIndexOffset]);
+
+ // Checks that the list is of ordered list type, and that the two indices are valid numeric values.
+ // e.g) 1.0, 2, 4.0 works, but 4.5, 3.2 would not.
+ if (!(listType == ATypeTag.ARRAY) || !ATypeHierarchy.isCompatible(oldIndexType, ATypeTag.DOUBLE)
+ || !ATypeHierarchy.isCompatible(newIndexType, ATypeTag.DOUBLE)) {
+ PointableHelper.setNull(result);
+ return;
+ }
+
+ listAccessor.reset(listBytes, offset);
+
+ AbstractCollectionType outputListType;
+
+ ATypeTag listItemTypeTag = listAccessor.getItemType();
+ IAType listItemType = TypeTagUtil.getBuiltinTypeByTag(listItemTypeTag);
+ if (listAccessor.getListType() == ATypeTag.ARRAY) {
+ outputListType = new AOrderedListType(listItemType, listItemType.getTypeName());
+ }
+ // Known list type, use it directly
+ else {
+ outputListType = (AbstractCollectionType) inputListType;
+ }
+
+ listBuilder.reset(outputListType);
+
+ try {
+
+ int listLen = listAccessor.size();
+
+ double oldIndexVal = ATypeHierarchy.getDoubleValue(funcIdentifier, 1, oldIndexBytes, oldIndexOffset);
+ double newIndexVal = ATypeHierarchy.getDoubleValue(funcIdentifier, 2, newIndexBytes, newIndexOffset);
+
+ //Checks that old/new indices are within the range of the list and whether they are valid values
+ if (Double.isNaN(oldIndexVal) || Double.isInfinite(oldIndexVal) || Math.floor(oldIndexVal) < oldIndexVal
+ || newIndexVal > (listLen - 1) || newIndexVal < -(listLen) || oldIndexVal < -(listLen)
+ || oldIndexVal > (listLen - 1)) {
+ PointableHelper.setNull(result);
+ return;
+ }
+
+ // Converting the indices values into integers to be used in iteration. Also accounting for the negative indices case by using modulo
+ int oldIndexInt = (int) oldIndexVal;
+ int newIndexInt = (int) newIndexVal;
+
+ // use modulus to account for negative indices case
+ oldIndexInt = (oldIndexInt + listLen) % listLen;
+ newIndexInt = (newIndexInt + listLen) % listLen;
+
+ // if no changes are to be made, then return original list
+ if (oldIndexInt == newIndexInt || listLen <= 1) {
+ result.set(listArg);
+ return;
+ }
+
+ buildList(oldIndexInt, newIndexInt, listLen, listAccessor, listBuilder);
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ storage.reset();
+ listBuilder.write(storage.getDataOutput(), true);
+ result.set(storage);
+ }
+
+ /**
+ *
+ * Default: Builds a list with the exact same values as the input list. Depending on the function that extends
+ * this abstract class, this method should be overridden to suit the extending function.
+ *
+ * @param oldIndexInt - Position of the item at the old index.
+ * @param newIndexInt - Position where the item at the old index wants to be.
+ * @param listLen - to iterate through the list
+ * @param listAccessor
+ * @param listBuilder
+ * @throws IOException
+ */
+ protected void buildList(int oldIndexInt, int newIndexInt, int listLen, ListAccessor listAccessor,
+ IAsterixListBuilder listBuilder) throws IOException {
+ for (int i = 0; i < listLen; i++) {
+ storage.reset();
+ listAccessor.writeItem(oldIndexInt, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+
+ }
+
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayBinarySearchDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayBinarySearchDescriptor.java
new file mode 100644
index 0000000..23e13d4
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayBinarySearchDescriptor.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
+
+import java.io.IOException;
+
+import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AMutableInt32;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.ListAccessor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
+import org.apache.asterix.runtime.utils.DescriptorFactoryUtil;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+/**
+ * array_binary_search(orderedList, searchValue) returns the index of the search value if it exists within the
+ * ordered list.
+ *
+ * It returns in order:
+ * Missing, if any of the arguments are missing.
+ * Null, if the arguments are null, if the list argument is not a list, or if the searchValue argument is not numerical.
+ * Otherwise, it returns the index of the first occurrence of the search value in the input list.
+ */
+
+public class ArrayBinarySearchDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ private IAType[] argTypes;
+
+ public static final IFunctionDescriptorFactory FACTORY = DescriptorFactoryUtil
+ .createFactory(ArrayBinarySearchDescriptor::new, FunctionTypeInferers.SET_ARGUMENTS_TYPE);
+
+ @Override
+ public void setImmutableStates(Object... states) {
+ argTypes = (IAType[]) states;
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ARRAY_BINARY_SEARCH;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException {
+ return new ArrayBinarySearchDescriptor.ArrayBinarySearchEval(args, ctx, argTypes);
+ }
+ };
+ }
+
+ public class ArrayBinarySearchEval implements IScalarEvaluator {
+
+ private final ArrayBackedValueStorage storage;
+ private final ArrayBackedValueStorage tempStorage;
+ private final IScalarEvaluator listArgEval;
+ private final IScalarEvaluator searchArgEval;
+ private final IPointable listArg;
+ private final IPointable searchArg;
+ private final IPointable tempVal;
+ private final IPointable tempVal2;
+ private final ListAccessor listAccessor;
+ private final IBinaryComparator comp;
+ private final ISerializerDeserializer<AInt32> serde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
+ private final AMutableInt32 resIndex = new AMutableInt32(0);
+
+ public ArrayBinarySearchEval(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx, IAType[] argTypes)
+ throws HyracksDataException {
+ storage = new ArrayBackedValueStorage();
+ tempStorage = new ArrayBackedValueStorage();
+ listArg = new VoidPointable();
+ searchArg = new VoidPointable();
+ tempVal = new VoidPointable();
+ tempVal2 = new VoidPointable();
+ listArgEval = args[0].createScalarEvaluator(ctx);
+ searchArgEval = args[1].createScalarEvaluator(ctx);
+ listAccessor = new ListAccessor();
+ comp = createComparator(argTypes[0], argTypes[1]);
+ }
+
+ private IBinaryComparator createComparator(IAType listType, IAType searchValueType) {
+ IAType itemType = listType.getTypeTag().isListType() ? ((AbstractCollectionType) listType).getItemType()
+ : BuiltinType.ANY;
+ return BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(itemType, searchValueType, true)
+ .createBinaryComparator();
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+
+ // argument missing/null checks
+ listArgEval.evaluate(tuple, listArg);
+ searchArgEval.evaluate(tuple, searchArg);
+ if (PointableHelper.checkAndSetMissingOrNull(result, listArg, searchArg)) {
+ return;
+ }
+
+ // Checking that our list arg is in fact a list
+ byte[] listBytes = listArg.getByteArray();
+ int offset = listArg.getStartOffset();
+ ATypeTag listType = ATYPETAGDESERIALIZER.deserialize(listBytes[offset]);
+
+ if (listType != ATypeTag.ARRAY) {
+ PointableHelper.setNull(result);
+ return;
+ }
+
+ byte[] searchBytes = searchArg.getByteArray();
+ int searchOffset = searchArg.getStartOffset();
+
+ listAccessor.reset(listBytes, offset);
+
+ int listLen = listAccessor.size();
+ int low = 0;
+ int high = listLen - 1;
+
+ try {
+ while (low <= high) {
+ int mid = low + ((high - low) / 2);
+ storage.reset();
+ listAccessor.getOrWriteItem(mid, tempVal, storage);
+ int comparison = comp.compare(tempVal.getByteArray(), tempVal.getStartOffset(), tempVal.getLength(),
+ searchBytes, searchOffset, searchArg.getLength());
+ if (comparison == 0) {
+ // if found, then find the first occurrence of the searchValue (from left to right)
+ int firstFoundIndex =
+ fetchFirstValue(mid, storage, tempStorage, listAccessor, comp, tempVal, tempVal2);
+ storage.reset();
+ resIndex.setValue(firstFoundIndex);
+ serde.serialize(resIndex, storage.getDataOutput());
+ result.set(storage);
+ return;
+ } else if (comparison < 0) {
+ low = mid + 1;
+ } else {
+ high = mid - 1;
+ }
+ }
+ storage.reset();
+ resIndex.setValue(-1);
+ serde.serialize(resIndex, storage.getDataOutput());
+ result.set(storage);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ }
+
+ private int fetchFirstValue(int midIndexArg, ArrayBackedValueStorage storage, ArrayBackedValueStorage storage2,
+ ListAccessor listAccessor, IBinaryComparator comp, IPointable tempVal1, IPointable tempVal2)
+ throws IOException {
+
+ int midIndex = midIndexArg;
+
+ if (midIndex == 0) {
+ return midIndex;
+ }
+ storage.reset();
+ listAccessor.getOrWriteItem(midIndex, tempVal1, storage);
+ storage.reset();
+ listAccessor.getOrWriteItem(midIndex - 1, tempVal2, storage2);
+ int prevComparison = comp.compare(tempVal1.getByteArray(), tempVal1.getStartOffset(), tempVal1.getLength(),
+ tempVal2.getByteArray(), tempVal2.getStartOffset(), tempVal2.getLength());
+ // If values before current value are not equal, then return current position.
+ if (prevComparison != 0) {
+ return midIndex;
+ } else {
+ // midIndex-1 position was already checked, so we now start checking the previous positions
+ midIndex--;
+ // to count the number of positions before the "midIndex" value to find first occurrence of search value.
+ int counter = 0;
+ while (prevComparison == 0) {
+ counter++;
+ if (midIndex - counter == 0) {
+ return 0;
+ }
+ storage2.reset();
+ listAccessor.getOrWriteItem(midIndex - counter, tempVal2, storage2);
+ prevComparison = comp.compare(tempVal1.getByteArray(), tempVal1.getStartOffset(), tempVal1.getLength(),
+ tempVal2.getByteArray(), tempVal2.getStartOffset(), tempVal2.getLength());
+ if (prevComparison != 0) {
+ return (midIndex - counter + 1);
+ }
+ }
+ }
+ return -1;
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayMoveDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayMoveDescriptor.java
new file mode 100644
index 0000000..82b5465
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayMoveDescriptor.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.IOException;
+
+import org.apache.asterix.builders.IAsterixListBuilder;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.ListAccessor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
+import org.apache.asterix.runtime.utils.DescriptorFactoryUtil;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+/**
+ * array_move(list, position1, position2) returns a new list, moving the item in position1 in such a way
+ * that now, it will be in position2. It will also move all other items accordingly.
+ *
+ * It returns in order:
+ * Missing, if any of the input arguments are missing.
+ * Null, if the arguments are null, if the list argument is not a list, or if the positional arguments are not numerical.
+ * Otherwise, it returns a new list, where the item at the old position is now in the new position, and all other
+ * items are moved accordingly.
+ */
+
+public class ArrayMoveDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ private IAType inputListType;
+ private String funcIDString = String.valueOf(BuiltinFunctions.ARRAY_MOVE);
+
+ public static final IFunctionDescriptorFactory FACTORY =
+ DescriptorFactoryUtil.createFactory(ArrayMoveDescriptor::new, FunctionTypeInferers.SET_ARGUMENTS_TYPE);
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ARRAY_MOVE;
+ }
+
+ @Override
+ public void setImmutableStates(Object... states) {
+ inputListType = (IAType) states[0];
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException {
+ return new ArrayMoveDescriptor.ArrayMoveEval(args, ctx);
+ }
+ };
+ }
+
+ public class ArrayMoveEval extends AbstractArrayMoveSwapEval {
+
+ private final ArrayBackedValueStorage storage;
+
+ ArrayMoveEval(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx) throws HyracksDataException {
+ super(args, ctx, funcIDString, inputListType);
+ storage = new ArrayBackedValueStorage();
+ }
+
+ @Override
+ protected void buildList(int oldIndexInt, int newIndexInt, int listLen, ListAccessor listAccessor,
+ IAsterixListBuilder listBuilder) throws IOException {
+
+ for (int i = 0; i < listLen; i++) {
+
+ if (oldIndexInt < newIndexInt) {
+ // if i outside of input indices range, just add items normally
+ if (i < oldIndexInt || i > newIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(i, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+ // if within range, but not equal to the new index, then shift the items down by 1
+ else if (i >= oldIndexInt && i < newIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(i + 1, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+ // if at new index position, then add the item that was at old index.
+ else {
+ storage.reset();
+ listAccessor.writeItem(oldIndexInt, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+ } else {
+ if (i < newIndexInt || i > oldIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(i, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ } else if (i > newIndexInt && i <= oldIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(i - 1, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ } else {
+ storage.reset();
+ listAccessor.writeItem(oldIndexInt, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArraySwapDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArraySwapDescriptor.java
new file mode 100644
index 0000000..279982c
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArraySwapDescriptor.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.IOException;
+
+import org.apache.asterix.builders.IAsterixListBuilder;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.ListAccessor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
+import org.apache.asterix.runtime.utils.DescriptorFactoryUtil;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+/**
+ * array_swap(list, position1, position2) returns a new list, switching the positions of both the items in position1
+ * and position2.
+ *
+ * It returns in order:
+ * Missing, if any of the input arguments are missing.
+ * Null, if the arguments are null, if the list argument is not a list, or if the positional arguments is not numerical.
+ * Otherwise, it returns a new list, where the two items at the
+ */
+
+public class ArraySwapDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ private IAType inputListType;
+ private String funcIDString = String.valueOf(BuiltinFunctions.ARRAY_SWAP);
+
+ public static final IFunctionDescriptorFactory FACTORY =
+ DescriptorFactoryUtil.createFactory(ArraySwapDescriptor::new, FunctionTypeInferers.SET_ARGUMENTS_TYPE);
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ARRAY_SWAP;
+ }
+
+ @Override
+ public void setImmutableStates(Object... states) {
+ inputListType = (IAType) states[0];
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException {
+ return new ArraySwapDescriptor.ArraySwapEval(args, ctx);
+ }
+ };
+ }
+
+ public class ArraySwapEval extends AbstractArrayMoveSwapEval {
+ private final ArrayBackedValueStorage storage;
+
+ ArraySwapEval(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx) throws HyracksDataException {
+ super(args, ctx, funcIDString, inputListType);
+ storage = new ArrayBackedValueStorage();
+ }
+
+ @Override
+ protected void buildList(int oldIndexInt, int newIndexInt, int listLen, ListAccessor listAccessor,
+ IAsterixListBuilder listBuilder) throws IOException {
+ for (int i = 0; i < listLen; i++) {
+ if (oldIndexInt < newIndexInt) {
+ if (i == oldIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(newIndexInt, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+ // the other case is when it is one of the two input indices, in which case we want to swap the two
+ else if (i == newIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(oldIndexInt, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ } else {
+ storage.reset();
+ listAccessor.writeItem(i, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+ } else if (oldIndexInt > newIndexInt) {
+ if (i == newIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(oldIndexInt, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ } else if (i == oldIndexInt) {
+ storage.reset();
+ listAccessor.writeItem(newIndexInt, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ } else {
+ storage.reset();
+ listAccessor.writeItem(i, storage.getDataOutput());
+ listBuilder.addItem(storage);
+ }
+ }
+ }
+ }
+
+ }
+
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/AbstractRecordAddPutEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/AbstractRecordAddPutEvaluator.java
index 47e84eb..ed7da3f 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/AbstractRecordAddPutEvaluator.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/AbstractRecordAddPutEvaluator.java
@@ -16,101 +16,64 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.runtime.evaluators.functions.records;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.builders.RecordBuilder;
-import org.apache.asterix.om.pointables.ARecordVisitablePointable;
-import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.evaluators.functions.CastTypeEvaluator;
import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.accessors.UTF8StringBinaryComparatorFactory;
import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-abstract class AbstractRecordAddPutEvaluator implements IScalarEvaluator {
-
- private final CastTypeEvaluator inputRecordCaster;
- private final CastTypeEvaluator argRecordCaster;
+abstract class AbstractRecordAddPutEvaluator extends AbstractRecordFunctionEvaluator {
private final IScalarEvaluator eval0;
private final IScalarEvaluator eval1;
private final IScalarEvaluator eval2;
- final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- final DataOutput resultOutput = resultStorage.getDataOutput();
- final IPointable inputRecordPointable = new VoidPointable();
- final UTF8StringPointable newFieldNamePointable = new UTF8StringPointable();
- final IPointable newFieldValuePointable = new VoidPointable();
- final IBinaryComparator stringBinaryComparator =
- UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- final RecordBuilder outRecordBuilder = new RecordBuilder();
- final ARecordVisitablePointable inputOpenRecordPointable;
- boolean newFieldValueIsMissing = false;
+ protected boolean newFieldValueIsMissing = false;
AbstractRecordAddPutEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, IScalarEvaluator eval2,
- IAType[] argTypes) {
+ ARecordType outRecType, ARecordType inRecType) {
+ super(outRecType, inRecType);
this.eval0 = eval0;
this.eval1 = eval1;
this.eval2 = eval2;
- inputOpenRecordPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
- inputRecordCaster = new CastTypeEvaluator(BuiltinType.ANY, argTypes[0], eval0);
- argRecordCaster = new CastTypeEvaluator(BuiltinType.ANY, argTypes[2], eval2);
}
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
resultStorage.reset();
- eval0.evaluate(tuple, inputRecordPointable);
+ eval0.evaluate(tuple, inputPointable);
eval1.evaluate(tuple, newFieldNamePointable);
eval2.evaluate(tuple, newFieldValuePointable);
- if (containsMissing(inputRecordPointable, newFieldNamePointable)) {
- writeTypeTag(ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
- result.set(resultStorage);
+ ATypeTag inputTypeTag = PointableHelper.getTypeTag(inputPointable);
+ ATypeTag newFieldNameTypeTag = PointableHelper.getTypeTag(newFieldNamePointable);
+ if (inputTypeTag == ATypeTag.MISSING || newFieldNameTypeTag == ATypeTag.MISSING) {
+ PointableHelper.setMissing(result);
return;
}
- final ATypeTag inputObjectType = PointableHelper.getTypeTag(inputRecordPointable);
- final ATypeTag newFieldNameValueType = PointableHelper.getTypeTag(newFieldNamePointable);
- if (inputObjectType != ATypeTag.OBJECT || newFieldNameValueType != ATypeTag.STRING) {
+ if (inputTypeTag != ATypeTag.OBJECT || newFieldNameTypeTag != ATypeTag.STRING) {
PointableHelper.setNull(result);
return;
}
- inputRecordCaster.evaluate(tuple, inputRecordPointable);
- final ATypeTag newFieldValueTag = PointableHelper.getTypeTag(newFieldValuePointable);
- if (newFieldValueTag.isDerivedType()) {
- argRecordCaster.evaluate(tuple, newFieldValuePointable);
+ newFieldValueIsMissing = PointableHelper.getTypeTag(newFieldValuePointable) == ATypeTag.MISSING;
+ outputRecordTypeInfo.reset(outRecType);
+ if (inputRecordPointable == null) {
+ inputRecordPointable = pointableAllocator.allocateRecordValue(inRecType);
}
- newFieldValueIsMissing = newFieldValueTag == ATypeTag.MISSING;
- buildOutputRecord();
+ buildOutputRecord(result);
result.set(resultStorage);
}
- protected abstract void buildOutputRecord() throws HyracksDataException;
+ protected abstract void buildOutputRecord(IPointable result) throws HyracksDataException;
- private boolean containsMissing(IPointable... pointables) {
- for (int i = 0; i < pointables.length; i++) {
- if (PointableHelper.getTypeTag(pointables[i]) == ATypeTag.MISSING) {
- return true;
- }
- }
- return false;
- }
-
- private void writeTypeTag(byte typeTag) throws HyracksDataException {
- try {
- resultOutput.writeByte(typeTag);
- } catch (IOException e) {
- throw HyracksDataException.create(e);
+ protected void addField(IPointable fieldName, IPointable fieldValue) throws HyracksDataException {
+ int pos = outputRecordTypeInfo.getFieldIndex(fieldName.getByteArray(), fieldName.getStartOffset() + 1,
+ fieldName.getLength() - 1);
+ if (pos >= 0) {
+ outRecordBuilder.addField(pos, fieldValue);
+ } else {
+ outRecordBuilder.addField(fieldName, fieldValue);
}
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/AbstractRecordFunctionEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/AbstractRecordFunctionEvaluator.java
new file mode 100644
index 0000000..2bb192d
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/AbstractRecordFunctionEvaluator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions.records;
+
+import java.io.DataOutput;
+
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.data.std.accessors.UTF8StringBinaryComparatorFactory;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+/**
+ * Base evaluator class for the following functions:
+ * 1. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_ADD}
+ * 2. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_PUT}
+ * 3. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_REMOVE}
+ * 4. {@link org.apache.asterix.om.functions.BuiltinFunctions#RECORD_RENAME}
+ */
+abstract class AbstractRecordFunctionEvaluator implements IScalarEvaluator {
+ protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ protected final DataOutput resultOutput = resultStorage.getDataOutput();
+ protected final RecordBuilder outRecordBuilder = new RecordBuilder();
+
+ protected final IPointable newFieldNamePointable = new VoidPointable();
+ protected final IPointable newFieldValuePointable = new VoidPointable();
+ protected final IBinaryComparator stringBinaryComparator =
+ UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+
+ protected final PointableAllocator pointableAllocator = new PointableAllocator();
+ protected final IPointable inputPointable = new VoidPointable();
+ protected final ARecordType inRecType;
+ protected ARecordVisitablePointable inputRecordPointable;
+
+ protected final ARecordType outRecType;
+ protected final RuntimeRecordTypeInfo outputRecordTypeInfo = new RuntimeRecordTypeInfo();
+
+ AbstractRecordFunctionEvaluator(ARecordType outRecType, ARecordType inRecType) {
+ this.outRecType = outRecType;
+ this.inRecType = inRecType;
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddDescriptor.java
index 67f2c8d..075b803 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddDescriptor.java
@@ -22,6 +22,8 @@
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.functions.IFunctionTypeInferer;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.asterix.runtime.functions.FunctionTypeInferers;
@@ -32,7 +34,6 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class RecordAddDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
@Override
public IFunctionDescriptor createFunctionDescriptor() {
@@ -41,19 +42,18 @@
@Override
public IFunctionTypeInferer createFunctionTypeInferer() {
- return FunctionTypeInferers.SET_ARGUMENTS_TYPE;
+ return FunctionTypeInferers.RECORD_MODIFY_INFERER;
}
};
private static final long serialVersionUID = 1L;
- private IAType[] argTypes;
+ private ARecordType outRecType;
+ private ARecordType inRecType;
@Override
public void setImmutableStates(Object... states) {
- argTypes = new IAType[states.length];
- for (int i = 0; i < states.length; i++) {
- argTypes[i] = (IAType) states[i];
- }
+ outRecType = TypeComputeUtils.extractRecordType((IAType) states[0]);
+ inRecType = TypeComputeUtils.extractRecordType((IAType) states[1]);
}
@Override
@@ -67,7 +67,7 @@
for (int i = 0; i < args.length; i++) {
argEvals[i] = args[i].createScalarEvaluator(ctx);
}
- return new RecordAddEvaluator(argEvals[0], argEvals[1], argEvals[2], argTypes);
+ return new RecordAddEvaluator(argEvals[0], argEvals[1], argEvals[2], outRecType, inRecType);
}
};
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddEvaluator.java
index 52a69bf..3df16d0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddEvaluator.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddEvaluator.java
@@ -16,42 +16,41 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.runtime.evaluators.functions.records;
import java.util.List;
-import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.pointables.base.IVisitablePointable;
-import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
class RecordAddEvaluator extends AbstractRecordAddPutEvaluator {
-
- RecordAddEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, IScalarEvaluator eval2, IAType[] argTypes) {
- super(eval0, eval1, eval2, argTypes);
+ RecordAddEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, IScalarEvaluator eval2, ARecordType outRecType,
+ ARecordType inRecType) {
+ super(eval0, eval1, eval2, outRecType, inRecType);
}
@Override
- protected void buildOutputRecord() throws HyracksDataException {
+ protected void buildOutputRecord(IPointable result) throws HyracksDataException {
resultStorage.reset();
- outRecordBuilder.reset(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
- outRecordBuilder.init();
- inputOpenRecordPointable.set(inputRecordPointable);
- final List<IVisitablePointable> fieldNames = inputOpenRecordPointable.getFieldNames();
- final List<IVisitablePointable> fieldValues = inputOpenRecordPointable.getFieldValues();
+ outRecordBuilder.reset(outRecType);
+ inputRecordPointable.set(inputPointable);
+ final List<IVisitablePointable> fieldNames = inputRecordPointable.getFieldNames();
+ final List<IVisitablePointable> fieldValues = inputRecordPointable.getFieldValues();
boolean newFieldFound = false;
for (int i = 0, fieldCount = fieldNames.size(); i < fieldCount; i++) {
final IVisitablePointable fieldName = fieldNames.get(i);
- if (PointableHelper.isEqual(fieldName, newFieldNamePointable, stringBinaryComparator)) {
+ final IVisitablePointable fieldValue = fieldValues.get(i);
+ if (!newFieldFound && PointableHelper.isEqual(fieldName, newFieldNamePointable, stringBinaryComparator)) {
newFieldFound = true;
}
- outRecordBuilder.addField(fieldName, fieldValues.get(i));
+ addField(fieldName, fieldValue);
}
if (!newFieldValueIsMissing && !newFieldFound) {
- outRecordBuilder.addField(newFieldNamePointable, newFieldValuePointable);
+ addField(newFieldNamePointable, newFieldValuePointable);
}
outRecordBuilder.write(resultOutput, true);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutDescriptor.java
index e4d72e5..870a2e8 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutDescriptor.java
@@ -22,6 +22,8 @@
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.functions.IFunctionTypeInferer;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.asterix.runtime.functions.FunctionTypeInferers;
@@ -32,7 +34,6 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class RecordPutDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
@Override
public IFunctionDescriptor createFunctionDescriptor() {
@@ -41,19 +42,18 @@
@Override
public IFunctionTypeInferer createFunctionTypeInferer() {
- return FunctionTypeInferers.SET_ARGUMENTS_TYPE;
+ return FunctionTypeInferers.RECORD_MODIFY_INFERER;
}
};
private static final long serialVersionUID = 1L;
- private IAType[] argTypes;
+ private ARecordType outRecType;
+ private ARecordType inRecType;
@Override
public void setImmutableStates(Object... states) {
- argTypes = new IAType[states.length];
- for (int i = 0; i < states.length; i++) {
- argTypes[i] = (IAType) states[i];
- }
+ outRecType = TypeComputeUtils.extractRecordType((IAType) states[0]);
+ inRecType = TypeComputeUtils.extractRecordType((IAType) states[1]);
}
@Override
@@ -67,7 +67,7 @@
for (int i = 0; i < args.length; i++) {
argEvals[i] = args[i].createScalarEvaluator(ctx);
}
- return new RecordPutEvaluator(argEvals[0], argEvals[1], argEvals[2], argTypes);
+ return new RecordPutEvaluator(argEvals[0], argEvals[1], argEvals[2], outRecType, inRecType);
}
};
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutEvaluator.java
index 857bd2e..3a9f3f8 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutEvaluator.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordPutEvaluator.java
@@ -16,51 +16,46 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.runtime.evaluators.functions.records;
import java.util.List;
-import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.pointables.base.IVisitablePointable;
-import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
class RecordPutEvaluator extends AbstractRecordAddPutEvaluator {
-
- RecordPutEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, IScalarEvaluator eval2, IAType[] argTypes) {
- super(eval0, eval1, eval2, argTypes);
+ RecordPutEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, IScalarEvaluator eval2, ARecordType outRecType,
+ ARecordType inRecType) {
+ super(eval0, eval1, eval2, outRecType, inRecType);
}
@Override
- protected void buildOutputRecord() throws HyracksDataException {
+ protected void buildOutputRecord(IPointable result) throws HyracksDataException {
resultStorage.reset();
- outRecordBuilder.reset(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
- outRecordBuilder.init();
- inputOpenRecordPointable.set(inputRecordPointable);
- final List<IVisitablePointable> fieldNames = inputOpenRecordPointable.getFieldNames();
- final List<IVisitablePointable> fieldValues = inputOpenRecordPointable.getFieldValues();
+ outRecordBuilder.reset(outRecType);
+ inputRecordPointable.set(inputPointable);
+ final List<IVisitablePointable> fieldNames = inputRecordPointable.getFieldNames();
+ final List<IVisitablePointable> fieldValues = inputRecordPointable.getFieldValues();
boolean newFieldFound = false;
for (int i = 0, fieldCount = fieldNames.size(); i < fieldCount; i++) {
final IVisitablePointable fieldName = fieldNames.get(i);
- if (!PointableHelper.isEqual(fieldName, newFieldNamePointable, stringBinaryComparator)) {
- outRecordBuilder.addField(fieldName, fieldValues.get(i));
+ final IVisitablePointable fieldValue = fieldValues.get(i);
+ if (!newFieldFound && !PointableHelper.isEqual(fieldName, newFieldNamePointable, stringBinaryComparator)) {
+ addField(fieldName, fieldValue);
} else {
newFieldFound = true;
if (!newFieldValueIsMissing) {
- putNewField();
+ addField(newFieldNamePointable, newFieldValuePointable);
}
}
}
if (!newFieldFound) {
- putNewField();
+ addField(newFieldNamePointable, newFieldValuePointable);
}
outRecordBuilder.write(resultOutput, true);
}
-
- private void putNewField() throws HyracksDataException {
- outRecordBuilder.addField(newFieldNamePointable, newFieldValuePointable);
- }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveDescriptor.java
index 8e44ada..818b650 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveDescriptor.java
@@ -23,7 +23,9 @@
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.functions.IFunctionTypeInferer;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.asterix.runtime.functions.FunctionTypeInferers;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
@@ -34,7 +36,6 @@
@MissingNullInOutFunction
public class RecordRemoveDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
@Override
public IFunctionDescriptor createFunctionDescriptor() {
@@ -43,16 +44,18 @@
@Override
public IFunctionTypeInferer createFunctionTypeInferer() {
- return FunctionTypeInferers.RecordAccessorTypeInferer.INSTANCE_LAX;
+ return FunctionTypeInferers.RECORD_MODIFY_INFERER;
}
};
private static final long serialVersionUID = 1L;
- private ARecordType recordType;
+ private ARecordType outRecType;
+ private ARecordType inRecType;
@Override
public void setImmutableStates(Object... states) {
- recordType = (ARecordType) states[0];
+ outRecType = TypeComputeUtils.extractRecordType((IAType) states[0]);
+ inRecType = TypeComputeUtils.extractRecordType((IAType) states[1]);
}
@Override
@@ -66,7 +69,7 @@
for (int i = 0; i < args.length; i++) {
argEvals[i] = args[i].createScalarEvaluator(ctx);
}
- return new RecordRemoveEvaluator(argEvals[0], argEvals[1], recordType);
+ return new RecordRemoveEvaluator(argEvals[0], argEvals[1], outRecType, inRecType);
}
};
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveEvaluator.java
index 51f66b0..e560c10 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveEvaluator.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveEvaluator.java
@@ -16,140 +16,87 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.runtime.evaluators.functions.records;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
-import org.apache.asterix.builders.RecordBuilder;
-import org.apache.asterix.om.pointables.ARecordVisitablePointable;
-import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.pointables.base.IVisitablePointable;
-import org.apache.asterix.om.pointables.cast.ACastVisitor;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
-import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.accessors.UTF8StringBinaryComparatorFactory;
import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-class RecordRemoveEvaluator implements IScalarEvaluator {
-
- private final IPointable inputRecordPointable = new VoidPointable();
- private final UTF8StringPointable fieldToRemovePointable = new UTF8StringPointable();
- private final IBinaryComparator stringBinaryComparator =
- UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput resultOutput = resultStorage.getDataOutput();
- private final RecordBuilder outRecordBuilder = new RecordBuilder();
+class RecordRemoveEvaluator extends AbstractRecordFunctionEvaluator {
private final IScalarEvaluator eval0;
private final IScalarEvaluator eval1;
- private final ARecordVisitablePointable openRecordPointable;
- private ARecordVisitablePointable inputRecordVisitable;
- private boolean requiresCast = false;
- private ACastVisitor castVisitor;
- private Triple<IVisitablePointable, IAType, Boolean> castVisitorArg;
+ private final IPointable fieldToRemovePointable = new VoidPointable();
- RecordRemoveEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, ARecordType recordType) {
+ RecordRemoveEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, ARecordType outRecType,
+ ARecordType inRecType) {
+ super(outRecType, inRecType);
this.eval0 = eval0;
this.eval1 = eval1;
- openRecordPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
- if (recordType != null) {
- inputRecordVisitable = new ARecordVisitablePointable(recordType);
- if (hasDerivedType(recordType.getFieldTypes())) {
- requiresCast = true;
- castVisitor = new ACastVisitor();
- castVisitorArg =
- new Triple<>(openRecordPointable, openRecordPointable.getInputRecordType(), Boolean.FALSE);
- }
- }
}
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
resultStorage.reset();
- boolean returnNull = false;
- eval0.evaluate(tuple, inputRecordPointable);
+ eval0.evaluate(tuple, inputPointable);
eval1.evaluate(tuple, fieldToRemovePointable);
-
- if (PointableHelper.checkAndSetMissingOrNull(result, inputRecordPointable, fieldToRemovePointable)) {
+ if (PointableHelper.checkAndSetMissingOrNull(result, inputPointable, fieldToRemovePointable)) {
return;
}
- byte[] data = inputRecordPointable.getByteArray();
- int offset = inputRecordPointable.getStartOffset();
+ // Check the type of our first argument.
+ byte[] data = inputPointable.getByteArray();
+ int offset = inputPointable.getStartOffset();
byte typeTag = data[offset];
if (typeTag != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
- returnNull = true;
+ PointableHelper.setNull(result);
+ return;
}
+ // Check the type of our second argument.
data = fieldToRemovePointable.getByteArray();
offset = fieldToRemovePointable.getStartOffset();
typeTag = data[offset];
if (typeTag != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
- returnNull = true;
- }
- if (returnNull) {
PointableHelper.setNull(result);
return;
}
- evaluate();
- result.set(resultStorage);
- }
- private void evaluate() throws HyracksDataException {
- resultStorage.reset();
try {
- final ARecordVisitablePointable inputRecord = getInputRecordVisitablePointable();
- buildOutputRecord(inputRecord);
+ outRecordBuilder.reset(outRecType);
+ outputRecordTypeInfo.reset(outRecType);
+ if (inputRecordPointable == null) {
+ inputRecordPointable = pointableAllocator.allocateRecordValue(inRecType);
+ }
+ inputRecordPointable.set(inputPointable);
+ final List<IVisitablePointable> fieldNames = inputRecordPointable.getFieldNames();
+ final List<IVisitablePointable> fieldValues = inputRecordPointable.getFieldValues();
+ for (int i = 0, fieldCount = fieldNames.size(); i < fieldCount; i++) {
+ final IVisitablePointable fieldName = fieldNames.get(i);
+ final IVisitablePointable fieldValue = fieldValues.get(i);
+ if (!PointableHelper.isEqual(fieldName, fieldToRemovePointable, stringBinaryComparator)) {
+ int pos = outputRecordTypeInfo.getFieldIndex(fieldName.getByteArray(),
+ fieldName.getStartOffset() + 1, fieldName.getLength() - 1);
+ if (pos >= 0) {
+ outRecordBuilder.addField(pos, fieldValue);
+ } else {
+ outRecordBuilder.addField(fieldName, fieldValue);
+ }
+ }
+ }
+ outRecordBuilder.write(resultOutput, true);
+
} catch (IOException e) {
throw HyracksDataException.create(e);
}
- }
-
- private void buildOutputRecord(ARecordVisitablePointable inputRecord) throws HyracksDataException {
- outRecordBuilder.reset(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
- outRecordBuilder.init();
- final List<IVisitablePointable> fieldNames = inputRecord.getFieldNames();
- final List<IVisitablePointable> fieldValues = inputRecord.getFieldValues();
- for (int i = 0, fieldCount = fieldNames.size(); i < fieldCount; i++) {
- final IVisitablePointable fieldName = fieldNames.get(i);
- if (!PointableHelper.isEqual(fieldName, fieldToRemovePointable, stringBinaryComparator)) {
- outRecordBuilder.addField(fieldName, fieldValues.get(i));
- }
- }
- outRecordBuilder.write(resultOutput, true);
- }
-
- private ARecordVisitablePointable getInputRecordVisitablePointable() throws HyracksDataException {
- inputRecordVisitable.set(inputRecordPointable);
- if (requiresCast) {
- return castToOpenRecord();
- }
- return inputRecordVisitable;
- }
-
- private boolean hasDerivedType(IAType[] types) {
- for (IAType type : types) {
- if (type.getTypeTag().isDerivedType()) {
- return true;
- }
- }
- return false;
- }
-
- private ARecordVisitablePointable castToOpenRecord() throws HyracksDataException {
- inputRecordVisitable.accept(castVisitor, castVisitorArg);
- return openRecordPointable;
+ result.set(resultStorage);
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java
index c8d4824..1a453a7 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java
@@ -33,6 +33,7 @@
import org.apache.asterix.om.pointables.PointableAllocator;
import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
@@ -170,8 +171,9 @@
int pos = runtimeRecordTypeInfo.getFieldIndex(fieldNamePointable.getByteArray(),
fieldNamePointable.getStartOffset() + 1, fieldNamePointable.getLength() - 1);
if (pos >= 0) { // Closed field
- if (PointableHelper.sameType(ATypeTag.OBJECT, fieldTypePointable)) {
- processRecord((ARecordType) requiredType.getFieldTypes()[pos],
+ if (PointableHelper.sameType(ATypeTag.OBJECT, fieldTypePointable)
+ && PointableHelper.sameType(ATypeTag.OBJECT, fieldValuePointable)) {
+ processRecord((ARecordType) TypeComputeUtils.getActualType(requiredType.getFieldTypes()[pos]),
(ARecordVisitablePointable) fieldValuePointable, inputList, nestedLevel + 1);
tabvs.reset();
rbStack.get(nestedLevel + 1).write(tabvs.getDataOutput(), true);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameDescriptor.java
index 8b345aa..ea4f6db 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameDescriptor.java
@@ -23,7 +23,9 @@
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.functions.IFunctionTypeInferer;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.asterix.runtime.functions.FunctionTypeInferers;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
@@ -34,7 +36,6 @@
@MissingNullInOutFunction
public class RecordRenameDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
@Override
public IFunctionDescriptor createFunctionDescriptor() {
@@ -43,16 +44,18 @@
@Override
public IFunctionTypeInferer createFunctionTypeInferer() {
- return FunctionTypeInferers.RecordAccessorTypeInferer.INSTANCE_LAX;
+ return FunctionTypeInferers.RECORD_MODIFY_INFERER;
}
};
private static final long serialVersionUID = 1L;
- private ARecordType recordType;
+ private ARecordType outRecType;
+ private ARecordType inRecType;
@Override
public void setImmutableStates(Object... states) {
- recordType = (ARecordType) states[0];
+ outRecType = TypeComputeUtils.extractRecordType((IAType) states[0]);
+ inRecType = TypeComputeUtils.extractRecordType((IAType) states[1]);
}
@Override
@@ -66,7 +69,7 @@
for (int i = 0; i < args.length; i++) {
argEvals[i] = args[i].createScalarEvaluator(ctx);
}
- return new RecordRenameEvaluator(argEvals[0], argEvals[1], argEvals[2], recordType);
+ return new RecordRenameEvaluator(argEvals[0], argEvals[1], argEvals[2], outRecType, inRecType);
}
};
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameEvaluator.java
index ac09ffa..1461536 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameEvaluator.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRenameEvaluator.java
@@ -16,156 +16,105 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.runtime.evaluators.functions.records;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
-import org.apache.asterix.builders.RecordBuilder;
-import org.apache.asterix.om.pointables.ARecordVisitablePointable;
-import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.pointables.base.IVisitablePointable;
-import org.apache.asterix.om.pointables.cast.ACastVisitor;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
-import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.accessors.UTF8StringBinaryComparatorFactory;
import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-class RecordRenameEvaluator implements IScalarEvaluator {
-
- private final IPointable inputRecordPointable = new VoidPointable();
- private final UTF8StringPointable oldFieldNamePointable = new UTF8StringPointable();
- private final UTF8StringPointable newFieldNamePointable = new UTF8StringPointable();
- private final IBinaryComparator stringBinaryComparator =
- UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput resultOutput = resultStorage.getDataOutput();
- private final RecordBuilder outRecordBuilder = new RecordBuilder();
+class RecordRenameEvaluator extends AbstractRecordFunctionEvaluator {
private final IScalarEvaluator eval0;
private final IScalarEvaluator eval1;
private final IScalarEvaluator eval2;
- private final ARecordVisitablePointable openRecordPointable;
- private ARecordVisitablePointable inputRecordVisitable;
- private boolean requiresCast = false;
- private ACastVisitor castVisitor;
- private Triple<IVisitablePointable, IAType, Boolean> castVisitorArg;
+ private final IPointable oldFieldNamePointable = new VoidPointable();
RecordRenameEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, IScalarEvaluator eval2,
- ARecordType recordType) {
+ ARecordType outRecType, ARecordType inRecType) {
+ super(outRecType, inRecType);
this.eval0 = eval0;
this.eval1 = eval1;
this.eval2 = eval2;
- openRecordPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
- if (recordType != null) {
- inputRecordVisitable = new ARecordVisitablePointable(recordType);
- if (hasDerivedType(recordType.getFieldTypes())) {
- requiresCast = true;
- castVisitor = new ACastVisitor();
- castVisitorArg =
- new Triple<>(openRecordPointable, openRecordPointable.getInputRecordType(), Boolean.FALSE);
- }
- }
}
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
resultStorage.reset();
- boolean returnNull = false;
- eval0.evaluate(tuple, inputRecordPointable);
+ eval0.evaluate(tuple, inputPointable);
eval1.evaluate(tuple, oldFieldNamePointable);
eval2.evaluate(tuple, newFieldNamePointable);
-
- if (PointableHelper.checkAndSetMissingOrNull(result, inputRecordPointable, oldFieldNamePointable,
+ if (PointableHelper.checkAndSetMissingOrNull(result, inputPointable, oldFieldNamePointable,
newFieldNamePointable)) {
return;
}
- byte[] data = inputRecordPointable.getByteArray();
- int offset = inputRecordPointable.getStartOffset();
+ // Check the type of our first argument.
+ byte[] data = inputPointable.getByteArray();
+ int offset = inputPointable.getStartOffset();
byte typeTag = data[offset];
if (typeTag != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
- returnNull = true;
+ PointableHelper.setNull(result);
+ return;
}
+ // Check the type of our second argument.
data = oldFieldNamePointable.getByteArray();
offset = oldFieldNamePointable.getStartOffset();
typeTag = data[offset];
if (typeTag != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
- returnNull = true;
+ PointableHelper.setNull(result);
+ return;
}
+ // Check the type of our third argument.
data = newFieldNamePointable.getByteArray();
offset = newFieldNamePointable.getStartOffset();
typeTag = data[offset];
if (typeTag != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
- returnNull = true;
- }
- if (returnNull) {
PointableHelper.setNull(result);
return;
}
- evaluate();
- result.set(resultStorage);
- }
- private void evaluate() throws HyracksDataException {
- resultStorage.reset();
try {
- final ARecordVisitablePointable inputRecord = getInputRecordVisitablePointable();
- buildOutputRecord(inputRecord);
+ outRecordBuilder.reset(outRecType);
+ outputRecordTypeInfo.reset(outRecType);
+ if (inputRecordPointable == null) {
+ inputRecordPointable = pointableAllocator.allocateRecordValue(inRecType);
+ }
+ inputRecordPointable.set(inputPointable);
+ final List<IVisitablePointable> fieldNames = inputRecordPointable.getFieldNames();
+ final List<IVisitablePointable> fieldValues = inputRecordPointable.getFieldValues();
+ for (int i = 0, fieldCount = fieldNames.size(); i < fieldCount; i++) {
+ final IVisitablePointable fieldName = fieldNames.get(i);
+ final IVisitablePointable fieldValue = fieldValues.get(i);
+ if (!PointableHelper.isEqual(fieldName, oldFieldNamePointable, stringBinaryComparator)) {
+ addField(fieldName, fieldValue);
+ } else {
+ addField(newFieldNamePointable, fieldValue);
+ }
+ }
+ outRecordBuilder.write(resultOutput, true);
} catch (IOException e) {
throw HyracksDataException.create(e);
}
+ result.set(resultStorage);
}
- private void buildOutputRecord(ARecordVisitablePointable inputRecord) throws HyracksDataException {
- outRecordBuilder.reset(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
- outRecordBuilder.init();
- final List<IVisitablePointable> fieldNames = inputRecord.getFieldNames();
- final List<IVisitablePointable> fieldValues = inputRecord.getFieldValues();
- for (int i = 0, fieldCount = fieldNames.size(); i < fieldCount; i++) {
- final IVisitablePointable fieldName = fieldNames.get(i);
- if (!PointableHelper.isEqual(fieldName, oldFieldNamePointable, stringBinaryComparator)) {
- outRecordBuilder.addField(fieldName, fieldValues.get(i));
- } else {
- outRecordBuilder.addField(newFieldNamePointable, fieldValues.get(i));
- }
+ private void addField(IPointable fieldName, IPointable fieldValue) throws HyracksDataException {
+ int pos = outputRecordTypeInfo.getFieldIndex(fieldName.getByteArray(), fieldName.getStartOffset() + 1,
+ fieldName.getLength() - 1);
+ if (pos >= 0) {
+ outRecordBuilder.addField(pos, fieldValue);
+ } else {
+ outRecordBuilder.addField(fieldName, fieldValue);
}
- outRecordBuilder.write(resultOutput, true);
}
-
- private ARecordVisitablePointable getInputRecordVisitablePointable() throws HyracksDataException {
- inputRecordVisitable.set(inputRecordPointable);
- if (requiresCast) {
- return castToOpenRecord();
- }
- return inputRecordVisitable;
- }
-
- private boolean hasDerivedType(IAType[] types) {
- for (IAType type : types) {
- if (type.getTypeTag().isDerivedType()) {
- return true;
- }
- }
- return false;
- }
-
- private ARecordVisitablePointable castToOpenRecord() throws HyracksDataException {
- inputRecordVisitable.accept(castVisitor, castVisitorArg);
- return openRecordPointable;
- }
-
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index 4b418b6..63dae91 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@ -327,6 +327,7 @@
import org.apache.asterix.runtime.evaluators.functions.AndDescriptor;
import org.apache.asterix.runtime.evaluators.functions.AnyCollectionMemberDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayAppendDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.ArrayBinarySearchDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayConcatDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayContainsDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayDistinctDescriptor;
@@ -335,6 +336,7 @@
import org.apache.asterix.runtime.evaluators.functions.ArrayIfNullDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayInsertDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayIntersectDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.ArrayMoveDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayPositionDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayPrependDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayPutDescriptor;
@@ -349,6 +351,7 @@
import org.apache.asterix.runtime.evaluators.functions.ArraySliceWithoutEndPositionDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArraySortDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayStarDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.ArraySwapDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArraySymDiffDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArraySymDiffnDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ArrayUnionDescriptor;
@@ -671,6 +674,9 @@
fc.add(ArraySymDiffnDescriptor.FACTORY);
fc.add(ArrayStarDescriptor.FACTORY);
fc.add(ArrayExceptDescriptor.FACTORY);
+ fc.add(ArrayMoveDescriptor.FACTORY);
+ fc.add(ArraySwapDescriptor.FACTORY);
+ fc.add(ArrayBinarySearchDescriptor.FACTORY);
// unnesting functions
fc.add(TidRunningAggregateDescriptor.FACTORY);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java
index bc763bd..5abebd2 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java
@@ -116,6 +116,16 @@
}
};
+ public static final IFunctionTypeInferer RECORD_MODIFY_INFERER = (expr, fd, context, compilerProps) -> {
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
+ IAType outType = (IAType) context.getType(expr);
+ IAType inType = (IAType) context.getType(f.getArguments().get(0).getValue());
+ if (inType.getTypeTag().equals(ATypeTag.ANY)) {
+ inType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ fd.setImmutableStates(outType, inType);
+ };
+
public static final class CastTypeInferer implements IFunctionTypeInferer {
@Override
public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context,
@@ -300,6 +310,8 @@
IAType type1 = (IAType) context.getType(le);
if (type0.getTypeTag().equals(ATypeTag.ANY)) {
type0 = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ } else if (type0.getTypeTag().equals(ATypeTag.UNION)) {
+ type0 = ((AUnionType) type0).getActualType();
}
if (type1.getTypeTag().equals(ATypeTag.ANY)) {
type1 = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java
index 8ea267a..16135ee 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java
@@ -37,14 +37,14 @@
public DatasetStreamStats(IOperatorStats opStats) {
this.cardinality = opStats.getTupleCounter().get();
- long totalTupleSize = opStats.getDiskIoCounter().get();
+ long totalTupleSize = opStats.getPageReads().get();
this.avgTupleSize = cardinality > 0 ? (int) (totalTupleSize / cardinality) : 0;
this.indexesStats = opStats.getIndexesStats();
}
static void update(IOperatorStats opStats, long tupleCount, long tupleSize, Map<String, IndexStats> indexStats) {
opStats.getTupleCounter().update(tupleCount);
- opStats.getDiskIoCounter().update(tupleSize);
+ opStats.getPageReads().update(tupleSize);
opStats.updateIndexesStats(indexStats);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
index 4f2e419..ef9e75b 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
@@ -134,7 +134,7 @@
public void close() throws HyracksDataException {
IStatsCollector statsCollector = ctx.getStatsCollector();
if (statsCollector != null) {
- IOperatorStats stats = statsCollector.getOrAddOperatorStats(operatorName);
+ IOperatorStats stats = statsCollector.getOperatorStats(operatorName);
DatasetStreamStats.update(stats, totalTupleCount, totalTupleLength, indexStats);
}
writer.close();
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/DataProjectionInfo.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/DataProjectionFiltrationInfo.java
similarity index 78%
rename from asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/DataProjectionInfo.java
rename to asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/DataProjectionFiltrationInfo.java
index de402ec..e625163 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/DataProjectionInfo.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/DataProjectionFiltrationInfo.java
@@ -29,9 +29,10 @@
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.visitor.SimpleStringBuilderForIATypeVisitor;
import org.apache.commons.lang3.SerializationUtils;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
-public class DataProjectionInfo implements IProjectionInfo<ARecordType> {
+public class DataProjectionFiltrationInfo implements IProjectionFiltrationInfo<ARecordType> {
//Default open record type when requesting the entire fields
public static final ARecordType ALL_FIELDS_TYPE = createType("");
//Default open record type when requesting none of the fields
@@ -39,13 +40,18 @@
private final ARecordType root;
private final Map<String, FunctionCallInformation> functionCallInfoMap;
+ private final Map<ILogicalExpression, ARecordType> expressionToPath;
+ private final ILogicalExpression filterExpression;
- public DataProjectionInfo(ARecordType root, Map<String, FunctionCallInformation> sourceInformationMap) {
+ public DataProjectionFiltrationInfo(ARecordType root, Map<String, FunctionCallInformation> sourceInformationMap,
+ Map<ILogicalExpression, ARecordType> expressionToPath, ILogicalExpression filterExpression) {
this.root = root;
this.functionCallInfoMap = sourceInformationMap;
+ this.expressionToPath = expressionToPath;
+ this.filterExpression = filterExpression;
}
- private DataProjectionInfo(DataProjectionInfo other) {
+ private DataProjectionFiltrationInfo(DataProjectionFiltrationInfo other) {
if (other.root == ALL_FIELDS_TYPE) {
root = ALL_FIELDS_TYPE;
} else if (other.root == EMPTY_TYPE) {
@@ -54,6 +60,8 @@
root = other.root.deepCopy(other.root);
}
functionCallInfoMap = new HashMap<>(other.functionCallInfoMap);
+ expressionToPath = new HashMap<>(other.expressionToPath);
+ filterExpression = other.filterExpression;
}
@Override
@@ -62,14 +70,23 @@
}
@Override
- public DataProjectionInfo createCopy() {
- return new DataProjectionInfo(this);
+ public DataProjectionFiltrationInfo createCopy() {
+ return new DataProjectionFiltrationInfo(this);
+ }
+
+ @Override
+ public ILogicalExpression getFilterExpression() {
+ return filterExpression;
}
public Map<String, FunctionCallInformation> getFunctionCallInfoMap() {
return functionCallInfoMap;
}
+ public Map<ILogicalExpression, ARecordType> getExpressionToPath() {
+ return expressionToPath;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -78,8 +95,10 @@
if (o == null || getClass() != o.getClass()) {
return false;
}
- DataProjectionInfo otherInfo = (DataProjectionInfo) o;
- return root.deepEqual(otherInfo.root) && Objects.equals(functionCallInfoMap, otherInfo.functionCallInfoMap);
+ DataProjectionFiltrationInfo otherInfo = (DataProjectionFiltrationInfo) o;
+ return root.deepEqual(otherInfo.root) && Objects.equals(functionCallInfoMap, otherInfo.functionCallInfoMap)
+ && Objects.equals(filterExpression, otherInfo.filterExpression)
+ && Objects.equals(expressionToPath, otherInfo.expressionToPath);
}
@Override
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/FunctionCallInformation.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/FunctionCallInformation.java
index 5cb26fd..65c5cd5 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/FunctionCallInformation.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/FunctionCallInformation.java
@@ -27,7 +27,6 @@
import java.util.Objects;
import java.util.Set;
-import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.om.exceptions.ExceptionUtil;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
@@ -41,16 +40,20 @@
private static final long serialVersionUID = -7884346933746232736L;
private final String functionName;
private final SourceLocation sourceLocation;
+ private final IProjectionFiltrationWarningFactory warningFactory;
private Set<ATypeTag> typeMismatches;
- public FunctionCallInformation(String functionName, SourceLocation sourceLocation) {
- this(functionName, sourceLocation, Collections.emptySet());
+ public FunctionCallInformation(String functionName, SourceLocation sourceLocation,
+ IProjectionFiltrationWarningFactory warningFactory) {
+ this(functionName, sourceLocation, Collections.emptySet(), warningFactory);
}
- private FunctionCallInformation(String functionName, SourceLocation sourceLocation, Set<ATypeTag> typeMismatches) {
+ private FunctionCallInformation(String functionName, SourceLocation sourceLocation, Set<ATypeTag> typeMismatches,
+ IProjectionFiltrationWarningFactory warningFactory) {
this.functionName = functionName;
this.sourceLocation = sourceLocation;
this.typeMismatches = typeMismatches;
+ this.warningFactory = warningFactory;
}
public String getFunctionName() {
@@ -61,16 +64,16 @@
return sourceLocation;
}
- public Warning createTypeMismatchWarning(ATypeTag expectedType, ATypeTag actualType) {
- if (typeMismatches == null) {
+ public Warning createWarning(ATypeTag expectedType, ATypeTag actualType) {
+ if (typeMismatches.isEmpty()) {
typeMismatches = EnumSet.noneOf(ATypeTag.class);
} else if (typeMismatches.contains(actualType)) {
//We already issued a warning containing the same actual type. So, we ignore it
return null;
}
typeMismatches.add(actualType);
- return Warning.of(getSourceLocation(), ErrorCode.TYPE_MISMATCH_FUNCTION, getFunctionName(),
- ExceptionUtil.indexToPosition(0), expectedType, actualType);
+ return warningFactory.createWarning(getSourceLocation(), getFunctionName(), ExceptionUtil.indexToPosition(0),
+ expectedType, actualType);
}
public void writeFields(DataOutput output) throws IOException {
@@ -86,11 +89,13 @@
String functionName = in.readUTF();
SourceLocation sourceLocation = SourceLocation.create(in);
int typeMismatchesLength = in.readInt();
- Set<ATypeTag> typeMismatches = EnumSet.noneOf(ATypeTag.class);
+ Set<ATypeTag> warnings = EnumSet.noneOf(ATypeTag.class);
+ IProjectionFiltrationWarningFactory warningFactory =
+ ProjectionFiltrationWarningFactoryProvider.TYPE_MISMATCH_FACTORY;
for (int i = 0; i < typeMismatchesLength; i++) {
- typeMismatches.add(ATypeTag.VALUE_TYPE_MAPPING[in.readByte()]);
+ warnings.add(ATypeTag.VALUE_TYPE_MAPPING[in.readByte()]);
}
- return new FunctionCallInformation(functionName, sourceLocation, typeMismatches);
+ return new FunctionCallInformation(functionName, sourceLocation, warnings, warningFactory);
}
@Override
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/IProjectionFiltrationWarningFactory.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/IProjectionFiltrationWarningFactory.java
new file mode 100644
index 0000000..4612f05
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/IProjectionFiltrationWarningFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.projection;
+
+import java.io.Serializable;
+
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.exceptions.Warning;
+
+public interface IProjectionFiltrationWarningFactory extends Serializable {
+ Warning createWarning(SourceLocation sourceLocation, String functionName, String position, ATypeTag expectedType,
+ ATypeTag actualType);
+
+ ErrorCode getErrorCode();
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ProjectionFiltrationWarningFactoryProvider.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ProjectionFiltrationWarningFactoryProvider.java
new file mode 100644
index 0000000..96de2cf
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ProjectionFiltrationWarningFactoryProvider.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.projection;
+
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.om.exceptions.ExceptionUtil;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.exceptions.Warning;
+
+public class ProjectionFiltrationWarningFactoryProvider {
+ private ProjectionFiltrationWarningFactoryProvider() {
+ }
+
+ public static final IProjectionFiltrationWarningFactory TYPE_MISMATCH_FACTORY =
+ new IProjectionFiltrationWarningFactory() {
+ private static final long serialVersionUID = 4263556611813387010L;
+
+ @Override
+ public Warning createWarning(SourceLocation sourceLocation, String functionName, String position,
+ ATypeTag expectedType, ATypeTag actualType) {
+ return Warning.of(sourceLocation, ErrorCode.TYPE_MISMATCH_FUNCTION, functionName,
+ ExceptionUtil.indexToPosition(0), expectedType, actualType);
+ }
+
+ @Override
+ public ErrorCode getErrorCode() {
+ return ErrorCode.TYPE_MISMATCH_FUNCTION;
+ }
+ };
+
+ public static IProjectionFiltrationWarningFactory getIncomparableTypesFactory(boolean leftConstant) {
+ return leftConstant ? LEFT_CONSTANT_INCOMPARABLE_TYPES_FACTORY : RIGHT_CONSTANT_INCOMPARABLE_TYPES_FACTORY;
+ }
+
+ private static final IProjectionFiltrationWarningFactory LEFT_CONSTANT_INCOMPARABLE_TYPES_FACTORY =
+ new IProjectionFiltrationWarningFactory() {
+ private static final long serialVersionUID = -7447187099851545763L;
+
+ @Override
+ public Warning createWarning(SourceLocation sourceLocation, String functionName, String position,
+ ATypeTag expectedType, ATypeTag actualType) {
+ return Warning.of(sourceLocation, ErrorCode.INCOMPARABLE_TYPES, expectedType, actualType);
+ }
+
+ @Override
+ public ErrorCode getErrorCode() {
+ return ErrorCode.INCOMPARABLE_TYPES;
+ }
+ };
+
+ private static final IProjectionFiltrationWarningFactory RIGHT_CONSTANT_INCOMPARABLE_TYPES_FACTORY =
+ new IProjectionFiltrationWarningFactory() {
+ private static final long serialVersionUID = 2818081955008928378L;
+
+ @Override
+ public Warning createWarning(SourceLocation sourceLocation, String functionName, String position,
+ ATypeTag expectedType, ATypeTag actualType) {
+ return Warning.of(sourceLocation, ErrorCode.INCOMPARABLE_TYPES, actualType, expectedType);
+ }
+
+ @Override
+ public ErrorCode getErrorCode() {
+ return ErrorCode.INCOMPARABLE_TYPES;
+ }
+ };
+}
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index fb1c2f1..54526df 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<licenses>
@@ -367,6 +367,12 @@
</override>
<override>
<gavs>
+ <gav>io.opencensus:opencensus-proto:0.2.0</gav>
+ </gavs>
+ <url>https://raw.githubusercontent.com/census-instrumentation/opencensus-proto/v0.2.0/LICENSE</url>
+ </override>
+ <override>
+ <gavs>
<gav>io.opencensus:opencensus-api:0.31.1</gav>
<gav>io.opencensus:opencensus-contrib-http-util:0.31.1</gav>
</gavs>
@@ -435,6 +441,23 @@
</override>
<override>
<gavs>
+ <gav>io.grpc:grpc-alts:1.43.2</gav>
+ <gav>io.grpc:grpc-api:1.43.2</gav>
+ <gav>io.grpc:grpc-auth:1.43.2</gav>
+ <gav>io.grpc:grpc-census:1.43.2</gav>
+ <gav>io.grpc:grpc-core:1.43.2</gav>
+ <gav>io.grpc:grpc-grpclb:1.43.2</gav>
+ <gav>io.grpc:grpc-protobuf-lite:1.43.2</gav>
+ <gav>io.grpc:grpc-protobuf:1.43.2</gav>
+ <gav>io.grpc:grpc-services:1.43.2</gav>
+ <gav>io.grpc:grpc-stub:1.43.2</gav>
+ <gav>io.grpc:grpc-xds:1.43.2</gav>
+ </gavs>
+ <url>https://raw.githubusercontent.com/grpc/grpc-java/v1.43.2/LICENSE</url>
+ <noticeUrl>https://raw.githubusercontent.com/grpc/grpc-java/v1.43.2/NOTICE.txt</noticeUrl>
+ </override>
+ <override>
+ <gavs>
<gav>io.grpc:grpc-alts:1.52.1</gav>
<gav>io.grpc:grpc-api:1.52.1</gav>
<gav>io.grpc:grpc-auth:1.52.1</gav>
@@ -477,6 +500,68 @@
<url>https://raw.githubusercontent.com/perfmark/perfmark/v0.26.0/LICENSE</url>
<noticeUrl>https://raw.githubusercontent.com/perfmark/perfmark/v0.26.0/NOTICE</noticeUrl>
</override>
+ <override>
+ <gavs>
+ <gav>com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.6</gav>
+ <gav>com.google.cloud.bigdataoss:gcsio:2.2.6</gav>
+ <gav>com.google.cloud.bigdataoss:util-hadoop:hadoop3-2.2.6</gav>
+ <gav>com.google.cloud.bigdataoss:util:2.2.6</gav>
+ </gavs>
+ <url>https://raw.githubusercontent.com/GoogleCloudDataproc/hadoop-connectors/v2.2.6/LICENSE</url>
+ </override>
+ <override>
+ <gavs>
+ <gav>com.google.flogger:flogger:0.7.1</gav>
+ <gav>com.google.flogger:google-extensions:0.7.1</gav>
+ <gav>com.google.flogger:flogger-system-backend:0.7.1</gav>
+ </gavs>
+ <url>https://raw.githubusercontent.com/google/flogger/flogger-0.7.1/LICENSE</url>
+ </override>
+ <override>
+ <gavs>
+ <gav>com.google.api.grpc:proto-google-cloud-monitoring-v3:1.64.0</gav>
+ <gav>com.google.api.grpc:proto-google-cloud-storage-v2:2.2.2-alpha</gav>
+ <gav>com.google.api.grpc:grpc-google-cloud-storage-v2:2.2.2-alpha</gav>
+ </gavs>
+ <url>https://raw.githubusercontent.com/googleapis/googleapis/master/LICENSE</url>
+ </override>
+ <override>
+ <gav>com.lmax:disruptor:3.4.2</gav>
+ <url>https://raw.githubusercontent.com/LMAX-Exchange/disruptor/3.4.2/LICENCE.txt</url>
+ </override>
+ <override>
+ <gav>com.google.cloud:google-cloud-core-grpc:1.82.0</gav>
+ <url>https://raw.githubusercontent.com/googleapis/java-core/v1.82.0/LICENSE</url>
+ </override>
+ <override>
+ <gav>com.google.cloud:google-cloud-monitoring:1.82.0</gav>
+ <url>https://raw.githubusercontent.com/googleapis/java-monitoring/1.82.0/LICENSE</url>
+ </override>
+ <override>
+ <gav>com.google.api-client:google-api-client-jackson2:1.32.2</gav>
+ <url>https://raw.githubusercontent.com/googleapis/google-api-java-client/v1.32.2/LICENSE</url>
+ </override>
+ <override>
+ <gav>org.conscrypt:conscrypt-openjdk-uber:2.5.1</gav>
+ <url>https://raw.githubusercontent.com/google/conscrypt/2.5.1/LICENSE</url>
+ <noticeUrl>https://raw.githubusercontent.com/google/conscrypt/2.5.1/NOTICE</noticeUrl>
+ </override>
+ <override>
+ <gav>io.perfmark:perfmark-api:0.23.0</gav>
+ <url>https://raw.githubusercontent.com/perfmark/perfmark/v0.23.0/LICENSE</url>
+ <noticeUrl>https://raw.githubusercontent.com/perfmark/perfmark/v0.23.0/NOTICE</noticeUrl>
+ </override>
+ <override>
+ <gav>com.google.api:gax-grpc:2.7.1</gav>
+ <url>https://raw.githubusercontent.com/googleapis/gax-java/v2.7.1/LICENSE</url>
+ </override>
+ <override>
+ <gavs>
+ <gav>org.bouncycastle:bcprov-jdk15on:1.60</gav>
+ <gav>org.bouncycastle:bcpkix-jdk15on:1.60</gav>
+ </gavs>
+ <url>https://raw.githubusercontent.com/bcgit/bc-java/r1rv60/LICENSE.html</url>
+ </override>
</overrides>
<licenses>
<license>
@@ -530,12 +615,26 @@
<aliasUrl>https://raw.githubusercontent.com/googleapis/googleapis/master/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/perfmark/perfmark/v0.26.0/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/googleapis/google-http-java-client/v1.42.3/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/google/conscrypt/2.5.1/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/google/flogger/flogger-0.7.1/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/googleapis/google-api-java-client/v1.32.2/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/googleapis/google-api-java-client/v1.35.1/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/googleapis/java-core/v1.82.0/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/googleapis/java-monitoring/1.82.0/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/GoogleCloudDataproc/hadoop-connectors/v2.2.6/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/LMAX-Exchange/disruptor/3.4.2/LICENCE.txt</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/perfmark/perfmark/v0.23.0/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/grpc/grpc-java/v1.43.2/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/grpc/grpc-java/v1.47.0/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/googleapis/google-http-java-client/v1.42.0/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/googleapis/google-oauth-java-client/v1.34.1/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/googleapis/java-core/v2.9.4/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/google/gson/gson-parent-2.10.1/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/googleapis/gapic-generator-java/v2.13.0/java-common-protos/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/googleapis/google-api-java-client/v2.1.2/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/grpc/grpc-java/v1.52.1/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/googleapis/java-core/v2.8.0/LICENSE</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/google/gson/gson-parent-2.9.0/LICENSE</aliasUrl>
</aliasUrls>
<metric>1</metric>
</license>
@@ -923,7 +1022,7 @@
<plugin>
<artifactId>jdeb</artifactId>
<groupId>org.vafer</groupId>
- <version>1.5</version>
+ <version>1.8</version>
<executions>
<execution>
<phase>package</phase>
@@ -933,26 +1032,36 @@
<configuration>
<dataSet>
<data>
- <src>${project.build.directory}/${project.build.finalName}-binary-assembly/apache-asterixdb-${project.version}/</src>
- <excludes>bin/**</excludes>
+ <src>${project.build.directory}/${project.build.finalName}-binary-assembly/apache-asterixdb-${project.version}</src>
<type>directory</type>
<mapper>
<type>perm</type>
- <prefix>/opt/apache-asterixdb-${project.version}/</prefix>
- <user>asterixdb</user>
- <group>asterixdb</group>
+ <prefix>/opt/apache-asterixdb/</prefix>
+ <user>root</user>
+ <group>root</group>
+ <filemode>755</filemode>
+ </mapper>
+ </data>
+ <data>
+ <type>file</type>
+ <src>src/deb/systemd/cc.conf</src>
+ <mapper>
+ <prefix>/opt/apache-asterixdb/</prefix>
+ <type>perm</type>
+ <user>root</user>
+ <group>root</group>
<filemode>644</filemode>
</mapper>
</data>
<data>
- <src>${project.build.directory}/${project.build.finalName}-binary-assembly/apache-asterixdb-${project.version}/bin</src>
- <type>directory</type>
+ <type>file</type>
+ <src>src/deb/udf_listener.py</src>
<mapper>
+ <prefix>/opt/apache-asterixdb/bin</prefix>
<type>perm</type>
- <prefix>/opt/apache-asterixdb-${project.version}/bin</prefix>
- <user>asterixdb</user>
- <group>asterixdb</group>
- <filemode>754</filemode>
+ <user>root</user>
+ <group>root</group>
+ <filemode>555</filemode>
</mapper>
</data>
<data>
@@ -975,6 +1084,39 @@
<group>root</group>
</mapper>
</data>
+ <data>
+ <type>file</type>
+ <src>src/deb/systemd/pyudf.socket</src>
+ <mapper>
+ <prefix>/lib/systemd/system</prefix>
+ <type>perm</type>
+ <user>root</user>
+ <group>root</group>
+ </mapper>
+ </data>
+ <data>
+ <type>file</type>
+ <src>src/deb/systemd/pyudf@.service</src>
+ <mapper>
+ <prefix>/lib/systemd/system</prefix>
+ <type>perm</type>
+ <user>root</user>
+ <group>root</group>
+ </mapper>
+ </data>
+ <data>
+ <type>template</type>
+ <paths>
+ <path>/opt/apache-asterixdb/logs</path>
+ <path>/opt/apache-asterixdb/data</path>
+ </paths>
+ <mapper>
+ <type>perm</type>
+ <user>asterixdb</user>
+ <group>asterixdb</group>
+ <filemode>750</filemode>
+ </mapper>
+ </data>
</dataSet>
</configuration>
</execution>
diff --git a/asterixdb/asterix-server/src/deb/control/control b/asterixdb/asterix-server/src/deb/control/control
index 1f6c213..77bbd1d 100644
--- a/asterixdb/asterix-server/src/deb/control/control
+++ b/asterixdb/asterix-server/src/deb/control/control
@@ -17,8 +17,7 @@
Section: databases
Priority: extra
Architecture: all
-Depends: jdk (>= 1.8)
+Depends: java17-runtime-headless
Maintainer: Ian Maxon <ian@maxons.email>
Description: Apache AsterixDB - a scalable, open source Big Data Management System (BDMS)
-Distribution: development
-Depends: default-jre | java8-runtime
+Distribution: development
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/deb/control/postinst b/asterixdb/asterix-server/src/deb/control/postinst
index 896ca28..fe5c912 100644
--- a/asterixdb/asterix-server/src/deb/control/postinst
+++ b/asterixdb/asterix-server/src/deb/control/postinst
@@ -13,5 +13,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-adduser --system --group --quiet --home /opt/apache-asterixdb/ \
---no-create-home --disabled-login --force-badname asterixdb
+chmod -R 755 /opt/apache-asterixdb/
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/deb/control/preinst b/asterixdb/asterix-server/src/deb/control/preinst
index 4509c90..8d14847 100644
--- a/asterixdb/asterix-server/src/deb/control/preinst
+++ b/asterixdb/asterix-server/src/deb/control/preinst
@@ -13,3 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+adduser --system --group --quiet --home /opt/apache-asterixdb/ \
+--no-create-home --disabled-login --force-badname asterixdb
+adduser --system --group --quiet --home /opt/apache-asterixdb/ \
+--no-create-home --disabled-login --force-badname asterixdb-udf
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/deb/systemd/asterix-cc.service b/asterixdb/asterix-server/src/deb/systemd/asterix-cc.service
index 9711fba..2a52e2d 100644
--- a/asterixdb/asterix-server/src/deb/systemd/asterix-cc.service
+++ b/asterixdb/asterix-server/src/deb/systemd/asterix-cc.service
@@ -19,8 +19,9 @@
[Service]
Type=simple
User=asterixdb
-ExecStart=/opt/apache-asterixdb/bin/asterixcc --config-file /opt/apache-asterixdb/cc.conf
+ExecStart=/opt/apache-asterixdb/bin/asterixcc -config-file "/opt/apache-asterixdb/cc.conf"
Restart=on-abort
+WorkingDirectory=/opt/apache-asterixdb
[Install]
WantedBy=multi-user.target
diff --git a/asterixdb/asterix-server/src/deb/systemd/asterix-nc.service b/asterixdb/asterix-server/src/deb/systemd/asterix-nc.service
index bfe6296..e09d8e8 100644
--- a/asterixdb/asterix-server/src/deb/systemd/asterix-nc.service
+++ b/asterixdb/asterix-server/src/deb/systemd/asterix-nc.service
@@ -21,6 +21,7 @@
User=asterixdb
ExecStart=/opt/apache-asterixdb/bin/asterixncservice
Restart=on-abort
+WorkingDirectory=/opt/apache-asterixdb
[Install]
WantedBy=multi-user.target
diff --git a/asterixdb/asterix-server/src/deb/systemd/cc.conf b/asterixdb/asterix-server/src/deb/systemd/cc.conf
new file mode 100644
index 0000000..0af967a
--- /dev/null
+++ b/asterixdb/asterix-server/src/deb/systemd/cc.conf
@@ -0,0 +1,33 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=/opt/apache-asterixdb/data/txnlog
+core.dump.dir=/opt/apache-asterixdb/logs/coredump
+iodevices=/opt/apache-asterixdb/data/
+nc.api.port=19004
+
+[nc]
+address=127.0.0.1
+command=asterixnc
+
+[cc]
+address = 127.0.0.1
+
+[common]
+log.level = INFO
+log.dir = /opt/apache-asterixdb/logs/
diff --git a/asterixdb/asterix-server/src/deb/systemd/pyudf.socket b/asterixdb/asterix-server/src/deb/systemd/pyudf.socket
new file mode 100644
index 0000000..4e731db
--- /dev/null
+++ b/asterixdb/asterix-server/src/deb/systemd/pyudf.socket
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[Unit]
+Description=AsterixDB UDF Domain Socket
+PartOf=asterixdb_udf.service
+
+[Socket]
+ListenStream=/tmp/pyudf.socket
+SocketMode=0660
+SocketUser=asterixdb-udf
+SocketGroup=asterixdb
+Accept=true
+DeferAcceptSec=1
+
+[Install]
+WantedBy=sockets.target
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/deb/systemd/pyudf@.service b/asterixdb/asterix-server/src/deb/systemd/pyudf@.service
new file mode 100644
index 0000000..9856142
--- /dev/null
+++ b/asterixdb/asterix-server/src/deb/systemd/pyudf@.service
@@ -0,0 +1,30 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[Unit]
+Description=AsterixDB UDF Executor Service
+After=network.target pyudf.socket
+Requires=pyudf.socket
+
+[Service]
+User=asterixdb-udf
+Type=simple
+ExecStart=/usr/bin/python3 /opt/apache-asterixdb/bin/udf_listener.py
+TimeoutStopSec=5
+StandardError=journal
+StandardError=journal
+
+[Install]
+WantedBy=default.target
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/deb/udf_listener.py b/asterixdb/asterix-server/src/deb/udf_listener.py
new file mode 100644
index 0000000..03874b2
--- /dev/null
+++ b/asterixdb/asterix-server/src/deb/udf_listener.py
@@ -0,0 +1,283 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+from systemd.daemon import listen_fds
+from os import chdir
+from os import getcwd
+from os import getpid
+from struct import *
+import signal
+import msgpack
+import socket
+import traceback
+from importlib import import_module
+from pathlib import Path
+from enum import IntEnum
+from io import BytesIO
+
+
+PROTO_VERSION = 1
+HEADER_SZ = 8 + 8 + 1
+REAL_HEADER_SZ = 4 + 8 + 8 + 1
+FRAMESZ = 32768
+
+
+class MessageType(IntEnum):
+ HELO = 0
+ QUIT = 1
+ INIT = 2
+ INIT_RSP = 3
+ CALL = 4
+ CALL_RSP = 5
+ ERROR = 6
+
+
+class MessageFlags(IntEnum):
+ NORMAL = 0
+ INITIAL_REQ = 1
+ INITIAL_ACK = 2
+ ERROR = 3
+
+
+class Wrapper(object):
+ wrapped_module = None
+ wrapped_class = None
+ wrapped_fn = None
+ sz = None
+ mid = None
+ rmid = None
+ flag = None
+ resp = None
+ unpacked_msg = None
+ msg_type = None
+ packer = msgpack.Packer(autoreset=False, use_bin_type=False)
+ unpacker = msgpack.Unpacker(raw=False)
+ response_buf = BytesIO()
+ stdin_buf = BytesIO()
+ wrapped_fns = {}
+ alive = True
+ readbuf = bytearray(FRAMESZ)
+ readview = memoryview(readbuf)
+
+
+ def init(self, module_name, class_name, fn_name):
+ self.wrapped_module = import_module(module_name)
+ # do not allow modules to be called that are not part of the uploaded module
+ wrapped_fn = None
+ if not self.check_module_path(self.wrapped_module):
+ self.wrapped_module = None
+ raise ImportError("Module was not found in library")
+ if class_name is not None:
+ self.wrapped_class = getattr(
+ import_module(module_name), class_name)()
+ if self.wrapped_class is not None:
+ wrapped_fn = getattr(self.wrapped_class, fn_name)
+ else:
+ wrapped_fn = getattr(import_module(module_name), fn_name)
+ if wrapped_fn is None:
+ raise ImportError(
+ "Could not find class or function in specified module")
+ self.wrapped_fns[self.mid] = wrapped_fn
+
+ def next_tuple(self, *args, key=None):
+ return self.wrapped_fns[key](*args)
+
+ def check_module_path(self, module):
+ cwd = Path('.').resolve()
+ module_path = Path(module.__file__).resolve()
+ return cwd in module_path.parents
+ return True
+
+ def read_header(self, readbuf):
+ self.sz, self.mid, self.rmid, self.flag = unpack(
+ "!iqqb", readbuf[0:REAL_HEADER_SZ])
+ return True
+
+ def write_header(self, response_buf, dlen):
+ total_len = dlen + HEADER_SZ
+ header = pack("!iqqb", total_len, int(-1), int(self.rmid), self.flag)
+ self.response_buf.write(header)
+ return total_len + 4
+
+ def get_ver_hlen(self, hlen):
+ return hlen + (PROTO_VERSION << 4)
+
+ def get_hlen(self):
+ return self.ver_hlen - (PROTO_VERSION << 4)
+
+ def init_remote_ipc(self):
+ self.response_buf.seek(0)
+ self.flag = MessageFlags.INITIAL_REQ
+ dlen = len(self.unpacked_msg[1])
+ resp_len = self.write_header(self.response_buf, dlen)
+ self.response_buf.write(self.unpacked_msg[1])
+ self.resp = self.response_buf.getbuffer()[0:resp_len]
+ self.send_msg()
+ self.packer.reset()
+
+ def cd(self, basedir):
+ chdir(basedir + "/site-packages")
+ sys.path.insert(0,getcwd())
+
+ def helo(self):
+ # need to ack the connection back before sending actual HELO
+ # self.init_remote_ipc()
+ self.cd(self.unpacked_msg[1][1])
+ self.flag = MessageFlags.NORMAL
+ self.response_buf.seek(0)
+ self.packer.pack(int(MessageType.HELO))
+ self.packer.pack(int(getpid()))
+ dlen = len(self.packer.bytes()) # tag(1) + body(4)
+ resp_len = self.write_header(self.response_buf, dlen)
+ self.response_buf.write(self.packer.bytes())
+ self.resp = self.response_buf.getbuffer()[0:resp_len]
+ self.send_msg()
+ self.packer.reset()
+ return True
+
+ def handle_init(self):
+ self.flag = MessageFlags.NORMAL
+ self.response_buf.seek(0)
+ args = self.unpacked_msg[1]
+ module = args[0]
+ if len(args) == 3:
+ clazz = args[1]
+ fn = args[2]
+ else:
+ clazz = None
+ fn = args[1]
+ self.init(module, clazz, fn)
+ self.packer.pack(int(MessageType.INIT_RSP))
+ dlen = 1 # just the tag.
+ resp_len = self.write_header(self.response_buf, dlen)
+ self.response_buf.write(self.packer.bytes())
+ self.resp = self.response_buf.getbuffer()[0:resp_len]
+ self.send_msg()
+ self.packer.reset()
+ return True
+
+ def quit(self):
+ self.alive = False
+ return True
+
+ def handle_call(self):
+ self.flag = MessageFlags.NORMAL
+ result = ([], [])
+ if len(self.unpacked_msg) > 1:
+ args = self.unpacked_msg[1]
+ if args is not None:
+ for arg in args:
+ try:
+ result[0].append(self.next_tuple(*arg, key=self.mid))
+ except BaseException as e:
+ result[1].append(traceback.format_exc())
+ self.packer.reset()
+ self.response_buf.seek(0)
+ body = msgpack.packb(result)
+ dlen = len(body) + 1 # 1 for tag
+ resp_len = self.write_header(self.response_buf, dlen)
+ self.packer.pack(int(MessageType.CALL_RSP))
+ self.response_buf.write(self.packer.bytes())
+ self.response_buf.write(body)
+ self.resp = self.response_buf.getbuffer()[0:resp_len]
+ self.send_msg()
+ self.packer.reset()
+ return True
+
+ def handle_error(self, e):
+ self.flag = MessageFlags.NORMAL
+ self.packer.reset()
+ self.response_buf.seek(0)
+ body = msgpack.packb(str(e))
+ dlen = len(body) + 1 # 1 for tag
+ resp_len = self.write_header(self.response_buf, dlen)
+ self.packer.pack(int(MessageType.ERROR))
+ self.response_buf.write(self.packer.bytes())
+ self.response_buf.write(body)
+ self.resp = self.response_buf.getbuffer()[0:resp_len]
+ self.send_msg()
+ self.packer.reset()
+ self.alive = False
+ return True
+
+ type_handler = {
+ MessageType.HELO: helo,
+ MessageType.QUIT: quit,
+ MessageType.INIT: handle_init,
+ MessageType.CALL: handle_call
+ }
+
+ def connect_sock(self):
+ self.sock = socket.fromfd(listen_fds()[0], socket.AF_UNIX, socket.SOCK_STREAM)
+
+ def disconnect_sock(self, *args):
+ self.sock.shutdown(socket.SHUT_RDWR)
+ self.sock.close()
+
+ def recv_msg(self):
+ while self.alive:
+ pos = self.sock.recv_into(self.readbuf)
+ if pos <= 0:
+ self.alive = False
+ return
+ try:
+ while pos < REAL_HEADER_SZ:
+ read = self.sock.recv_into(self.readview[pos:])
+ if read <= 0:
+ self.alive = False
+ return
+ pos += read
+ self.read_header(self.readview)
+ while pos < self.sz and len(self.readbuf) - pos > 0:
+ read = self.sock.recv_into(self.readview[pos:])
+ if read <= 0:
+ self.alive = False
+ return
+ pos += read
+ while pos < self.sz:
+ vszchunk = self.sock.recv(4096)
+ if len(vszchunk) == 0:
+ self.alive = False
+ return
+ self.readview.release()
+ self.readbuf.extend(vszchunk)
+ self.readview = memoryview(self.readbuf)
+ pos += len(vszchunk)
+ self.unpacker.feed(self.readview[REAL_HEADER_SZ:self.sz])
+ self.unpacked_msg = list(self.unpacker)
+ self.msg_type = MessageType(self.unpacked_msg[0])
+ self.type_handler[self.msg_type](self)
+ except BaseException as e:
+ self.handle_error(''.join(traceback.format_exc()))
+
+ def send_msg(self):
+ self.sock.sendall(self.resp)
+ self.resp = None
+ return
+
+ def recv_loop(self):
+ while self.alive:
+ self.recv_msg()
+ self.disconnect_sock()
+
+
+wrap = Wrapper()
+wrap.connect_sock()
+signal.signal(signal.SIGTERM, wrap.disconnect_sock)
+wrap.recv_loop()
diff --git a/asterixdb/asterix-spidersilk/pom.xml b/asterixdb/asterix-spidersilk/pom.xml
index 656501f..873d321 100644
--- a/asterixdb/asterix-spidersilk/pom.xml
+++ b/asterixdb/asterix-spidersilk/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/asterixdb/asterix-test-framework/pom.xml b/asterixdb/asterix-test-framework/pom.xml
index c150fae..65a9989 100644
--- a/asterixdb/asterix-test-framework/pom.xml
+++ b/asterixdb/asterix-test-framework/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-test-framework</artifactId>
diff --git a/asterixdb/asterix-tools/pom.xml b/asterixdb/asterix-tools/pom.xml
index 2924fc6..f8c98ec 100644
--- a/asterixdb/asterix-tools/pom.xml
+++ b/asterixdb/asterix-tools/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-tools</artifactId>
diff --git a/asterixdb/asterix-transactions/pom.xml b/asterixdb/asterix-transactions/pom.xml
index 2749096..4c92fa9 100644
--- a/asterixdb/asterix-transactions/pom.xml
+++ b/asterixdb/asterix-transactions/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-asterixdb</artifactId>
<groupId>org.apache.asterix</groupId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
</parent>
<artifactId>asterix-transactions</artifactId>
<licenses>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index c89e7e4..1c9db43 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.asterix</groupId>
<artifactId>apache-asterixdb</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
<packaging>pom</packaging>
<url>${implementation.url}</url>
@@ -82,8 +82,8 @@
<skip-npm-touch.stage>none</skip-npm-touch.stage>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <algebricks.version>0.3.8-SNAPSHOT</algebricks.version>
- <hyracks.version>0.3.8-SNAPSHOT</hyracks.version>
+ <algebricks.version>0.3.9-SNAPSHOT</algebricks.version>
+ <hyracks.version>0.3.9-SNAPSHOT</hyracks.version>
<hadoop.version>3.3.4</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.19.0</log4j.version>
@@ -94,6 +94,7 @@
<azuredatalakejavasdk.version>12.7.2</azuredatalakejavasdk.version>
<gcsjavasdk.version>2.17.2</gcsjavasdk.version>
<hadoop-azuresdk.version>8.6.6</hadoop-azuresdk.version>
+ <hadoop-gcs.version>hadoop3-2.2.6</hadoop-gcs.version>
<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
@@ -930,7 +931,7 @@
<module>asterix-test-framework</module>
<module>asterix-maven-plugins</module>
<module>asterix-server</module>
- <module>asterix-docker</module>
+ <module>asterix-podman</module>
<module>asterix-doc</module>
<module>asterix-fuzzyjoin</module>
<module>asterix-replication</module>
@@ -941,6 +942,7 @@
<module>asterix-license</module>
<module>asterix-geo</module>
<module>asterix-spidersilk</module>
+ <module>asterix-column</module>
</modules>
<dependencyManagement>
@@ -1383,6 +1385,11 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree-column</artifactId>
+ <version>${hyracks.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-rtree</artifactId>
<version>${hyracks.version}</version>
</dependency>
@@ -1896,6 +1903,28 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-common</artifactId>
+ <version>${parquet.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-encoding</artifactId>
+ <version>${parquet.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.kitesdk</groupId>
<artifactId>kite-data-core</artifactId>
<version>1.1.0</version>
@@ -1966,6 +1995,19 @@
</exclusions>
</dependency>
<!-- Hadoop Azure end -->
+ <!-- Hadoop GCS start -->
+ <dependency>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ <version>${hadoop-gcs.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-compat-qual</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Hadoop GCS end -->
<!-- TODO(htowaileb): removed from hadoop transitively and added separately to avoid CVEs, can
be removed once upgraded to hadoop 3.3.4 as it addresses the CVEs -->
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index 18508ef6..ae9f9fd 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -1198,6 +1198,31 @@
</project>
</supplement>
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-proto</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.2.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.2.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.2.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.android:annotations has ASLv2 <license> in pom -->
+ <supplement>
+ <project>
+ <groupId>com.google.android</groupId>
+ <artifactId>annotations</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>4.1.1.4</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>4.1.1.4</license.ignoreMissingEmbeddedNotice>
+ </properties>
+ </project>
+ </supplement>
+
<!-- io.opencensus uses non-fixed ALv2 with no NOTICE file -->
<supplement>
<project>
@@ -1237,6 +1262,32 @@
</project>
</supplement>
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-contrib-exemplar-util</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.31.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.31.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.31.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-contrib-grpc-metrics</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.31.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.31.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.31.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
<!-- io.opencensus uses non-fixed ALv2 with no NOTICE file -->
<supplement>
<project>
@@ -1365,6 +1416,239 @@
</project>
</supplement>
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-contrib-resource-util</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.31.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.31.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.31.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-exporter-metrics-util</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.31.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.31.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.31.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-exporter-stats-stackdriver</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.31.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.31.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.31.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-impl-core</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.31.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.31.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.31.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.opencensus uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-impl</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.31.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.31.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.31.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.cloud.bigdataoss uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>hadoop3-2.2.6</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>hadoop3-2.2.6</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>hadoop3-2.2.6</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.cloud.bigdataoss uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcsio</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>2.2.6</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>2.2.6</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>2.2.6</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.cloud.bigdataoss uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>util-hadoop</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>hadoop3-2.2.6</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>hadoop3-2.2.6</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>hadoop3-2.2.6</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.cloud.bigdataoss uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>util</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>2.2.6</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>2.2.6</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>2.2.6</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.flogger uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.flogger</groupId>
+ <artifactId>flogger</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.7.1</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.7.1</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.7.1</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.flogger uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.flogger</groupId>
+ <artifactId>google-extensions</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.7.1</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.7.1</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.7.1</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.flogger uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.flogger</groupId>
+ <artifactId>flogger-system-backend</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.7.1</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.7.1</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.7.1</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.api.grpc uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-cloud-monitoring-v3</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.64.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.64.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.64.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.lmax uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>3.4.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>3.4.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>3.4.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.cloud uses ALv2 with no NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-monitoring</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.82.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.82.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.82.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.api-client uses ALv2 LICENSE and has a NOTICE file -->
+ <supplement>
+ <project>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client-jackson2</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.32.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.32.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.32.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.perfmark uses ALv2 license -->
+ <supplement>
+ <project>
+ <groupId>io.perfmark</groupId>
+ <artifactId>perfmark-api</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.23.0</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.23.0</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>0.23.0</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- com.google.apis:google-api-services-iamcredentials has embedded ASLv2 in pom.xml -->
+ <supplement>
+ <project>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-iamcredentials</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>v1-rev20210326-1.32.1</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>v1-rev20210326-1.32.1</license.ignoreMissingEmbeddedNotice>
+ </properties>
+ </project>
+ </supplement>
+
<!-- com.google.cloud:google-cloud-core is non-fixed ALv2 with no NOTICE file -->
<supplement>
<project>
@@ -1436,9 +1720,10 @@
<groupId>io.grpc</groupId>
<artifactId>grpc-api</artifactId>
<properties>
- <license.ignoreMissingEmbeddedLicense>1.52.1</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>1.52.1</license.ignoreMissingEmbeddedNotice>
- <license.ignoreLicenseOverride>1.52.1</license.ignoreLicenseOverride>
+ <license.ignoreMissingEmbeddedLicense>1.43.2,1.52.1</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2,1.52.1</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2,1.52.1</license.ignoreLicenseOverride>
+ <license.ignoreNoticeOverride>1.43.2</license.ignoreNoticeOverride>
</properties>
</project>
</supplement>
@@ -1587,6 +1872,97 @@
</project>
</supplement>
+ <!-- io.grpc uses ALv2 -->
+ <supplement>
+ <project>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.43.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.grpc uses ALv2 -->
+ <supplement>
+ <project>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-grpclb</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.43.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.grpc uses ALv2 -->
+ <supplement>
+ <project>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-lite</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.43.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.grpc uses ALv2 -->
+ <supplement>
+ <project>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.43.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.grpc uses ALv2 -->
+ <supplement>
+ <project>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-services</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.43.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.grpc uses ALv2 -->
+ <supplement>
+ <project>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.43.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <!-- io.grpc uses ALv2 -->
+ <supplement>
+ <project>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-xds</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>1.43.2</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>1.43.2</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.43.2</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
<!-- com.google.protobuf has no NOTICE file -->
<supplement>
<project>
@@ -2072,4 +2448,28 @@
</properties>
</project>
</supplement>
+
+ <supplement>
+ <project>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedNotice>1.60</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreMissingEmbeddedLicense>1.60</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreLicenseOverride>1.60</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
+
+ <supplement>
+ <project>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedNotice>1.60</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreMissingEmbeddedLicense>1.60</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreLicenseOverride>1.60</license.ignoreLicenseOverride>
+ </properties>
+ </project>
+ </supplement>
</supplementalDataModels>
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_bcgit_bc-java_r1rv60_LICENSE.html.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_bcgit_bc-java_r1rv60_LICENSE.html.txt
new file mode 100644
index 0000000..b8f46c5
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_bcgit_bc-java_r1rv60_LICENSE.html.txt
@@ -0,0 +1,17 @@
+Copyright (c) 2000-2018 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_google_conscrypt_2.5.1_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_google_conscrypt_2.5.1_NOTICE.txt
new file mode 100644
index 0000000..80715a6
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_google_conscrypt_2.5.1_NOTICE.txt
@@ -0,0 +1,30 @@
+Copyright 2016 The Android Open Source Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-----------------------------------------------------------------------
+This product contains a modified portion of `Netty`, a configurable network
+stack in Java, which can be obtained at:
+
+ * LICENSE:
+ * licenses/LICENSE.netty.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * http://netty.io/
+
+This product contains a modified portion of `Apache Harmony`, modular Java runtime,
+which can be obtained at:
+
+ * LICENSE:
+ * licenses/LICENSE.harmony.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * https://harmony.apache.org/
\ No newline at end of file
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_googleapis_gax-java_v2.7.1_LICENSE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_googleapis_gax-java_v2.7.1_LICENSE.txt
new file mode 100644
index 0000000..6874140
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_googleapis_gax-java_v2.7.1_LICENSE.txt
@@ -0,0 +1,27 @@
+Copyright 2016, Google Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_grpc_grpc-java_v1.43.2_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_grpc_grpc-java_v1.43.2_NOTICE.txt
new file mode 100644
index 0000000..c5d3ec2
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_grpc_grpc-java_v1.43.2_NOTICE.txt
@@ -0,0 +1,62 @@
+Copyright 2014 The gRPC Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-----------------------------------------------------------------------
+
+This product contains a modified portion of 'OkHttp', an open source
+HTTP & SPDY client for Android and Java applications, which can be obtained
+at:
+
+ * LICENSE:
+ * okhttp/third_party/okhttp/LICENSE (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/square/okhttp
+ * LOCATION_IN_GRPC:
+ * okhttp/third_party/okhttp
+
+This product contains a modified portion of 'Envoy', an open source
+cloud-native high-performance edge/middle/service proxy, which can be
+obtained at:
+
+ * LICENSE:
+ * xds/third_party/envoy/LICENSE (Apache License 2.0)
+ * NOTICE:
+ * xds/third_party/envoy/NOTICE
+ * HOMEPAGE:
+ * https://www.envoyproxy.io
+ * LOCATION_IN_GRPC:
+ * xds/third_party/envoy
+
+This product contains a modified portion of 'protoc-gen-validate (PGV)',
+an open source protoc plugin to generate polyglot message validators,
+which can be obtained at:
+
+ * LICENSE:
+ * xds/third_party/protoc-gen-validate/LICENSE (Apache License 2.0)
+ * NOTICE:
+ * xds/third_party/protoc-gen-validate/NOTICE
+ * HOMEPAGE:
+ * https://github.com/envoyproxy/protoc-gen-validate
+ * LOCATION_IN_GRPC:
+ * xds/third_party/protoc-gen-validate
+
+This product contains a modified portion of 'udpa',
+an open source universal data plane API, which can be obtained at:
+
+ * LICENSE:
+ * xds/third_party/udpa/LICENSE (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/cncf/udpa
+ * LOCATION_IN_GRPC:
+ * xds/third_party/udpa
\ No newline at end of file
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_perfmark_perfmark_v0.23.0_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_perfmark_perfmark_v0.23.0_NOTICE.txt
new file mode 100644
index 0000000..3ed2d12
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_perfmark_perfmark_v0.23.0_NOTICE.txt
@@ -0,0 +1,32 @@
+
+Copyright 2019 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-----------------------------------------------------------------------
+
+This product contains a modified portion of 'Catapult', an open source
+Trace Event viewer for Chome, Linux, and Android applications, which can
+be obtained at:
+
+ * LICENSE:
+ * traceviewer/src/main/resources/io/perfmark/traceviewer/third_party/catapult/LICENSE (New BSD License)
+ * HOMEPAGE:
+ * https://github.com/catapult-project/catapult
+
+This product contains a modified portion of 'Polymer', a library for Web
+Components, which can be obtained at:
+ * LICENSE:
+ * traceviewer/src/main/resources/io/perfmark/traceviewer/third_party/polymer/LICENSE (New BSD License)
+ * HOMEPAGE:
+ * https://github.com/Polymer/polymer
\ No newline at end of file
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_typetools_checker-framework_checker-framework-2.5.3_LICENSE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_typetools_checker-framework_checker-framework-2.5.3_LICENSE.txt
new file mode 100644
index 0000000..fe8c705
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_typetools_checker-framework_checker-framework-2.5.3_LICENSE.txt
@@ -0,0 +1,408 @@
+Most of the Checker Framework is licensed under the GNU General Public
+License, version 2 (GPL2), with the classpath exception. The text of this
+license appears below. This is the same license used for OpenJDK.
+
+A few parts of the Checker Framework have more permissive licenses.
+
+ * The annotations are licensed under the MIT License. (The text of this
+ license appears below.) More specifically, all the parts of the Checker
+ Framework that you might want to include with your own program use the
+ MIT License. This is the checker-qual.jar file and all the files that
+ appear in it: every file in a qual/ directory, plus utility files such
+ as NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc.
+ In addition, the cleanroom implementations of third-party annotations,
+ which the Checker Framework recognizes as aliases for its own
+ annotations, are licensed under the MIT License.
+
+Some external libraries that are included with the Checker Framework have
+different licenses.
+
+ * javaparser is dual licensed under the LGPL or the Apache license -- you
+ may use it under whichever one you want. (The javaparser source code
+ contains a file with the text of the GPL, but it is not clear why, since
+ javaparser does not use the GPL.) See file stubparser/LICENSE
+ and the source code of all its files.
+
+ * JUnit is licensed under the Common Public License v1.0 (see
+ http://www.junit.org/license), with parts (Hamcrest) licensed under the
+ BSD License (see http://hamcrest.org/JavaHamcrest/).
+
+ * plume-lib is licensed under the MIT License.
+
+The Checker Framework includes annotations for the JDK in directory
+checker/jdk/, and for some other libraries. Each annotated library uses
+the same license as the unannotated version of the library.
+
+===========================================================================
+
+The GNU General Public License (GPL)
+
+Version 2, June 1991
+
+Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this license
+document, but changing it is not allowed.
+
+Preamble
+
+The licenses for most software are designed to take away your freedom to share
+and change it. By contrast, the GNU General Public License is intended to
+guarantee your freedom to share and change free software--to make sure the
+software is free for all its users. This General Public License applies to
+most of the Free Software Foundation's software and to any other program whose
+authors commit to using it. (Some other Free Software Foundation software is
+covered by the GNU Library General Public License instead.) You can apply it to
+your programs, too.
+
+When we speak of free software, we are referring to freedom, not price. Our
+General Public Licenses are designed to make sure that you have the freedom to
+distribute copies of free software (and charge for this service if you wish),
+that you receive source code or can get it if you want it, that you can change
+the software or use pieces of it in new free programs; and that you know you
+can do these things.
+
+To protect your rights, we need to make restrictions that forbid anyone to deny
+you these rights or to ask you to surrender the rights. These restrictions
+translate to certain responsibilities for you if you distribute copies of the
+software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether gratis or for
+a fee, you must give the recipients all the rights that you have. You must
+make sure that they, too, receive or can get the source code. And you must
+show them these terms so they know their rights.
+
+We protect your rights with two steps: (1) copyright the software, and (2)
+offer you this license which gives you legal permission to copy, distribute
+and/or modify the software.
+
+Also, for each author's protection and ours, we want to make certain that
+everyone understands that there is no warranty for this free software. If the
+software is modified by someone else and passed on, we want its recipients to
+know that what they have is not the original, so that any problems introduced
+by others will not reflect on the original authors' reputations.
+
+Finally, any free program is threatened constantly by software patents. We
+wish to avoid the danger that redistributors of a free program will
+individually obtain patent licenses, in effect making the program proprietary.
+To prevent this, we have made it clear that any patent must be licensed for
+everyone's free use or not licensed at all.
+
+The precise terms and conditions for copying, distribution and modification
+follow.
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. This License applies to any program or other work which contains a notice
+placed by the copyright holder saying it may be distributed under the terms of
+this General Public License. The "Program", below, refers to any such program
+or work, and a "work based on the Program" means either the Program or any
+derivative work under copyright law: that is to say, a work containing the
+Program or a portion of it, either verbatim or with modifications and/or
+translated into another language. (Hereinafter, translation is included
+without limitation in the term "modification".) Each licensee is addressed as
+"you".
+
+Activities other than copying, distribution and modification are not covered by
+this License; they are outside its scope. The act of running the Program is
+not restricted, and the output from the Program is covered only if its contents
+constitute a work based on the Program (independent of having been made by
+running the Program). Whether that is true depends on what the Program does.
+
+1. You may copy and distribute verbatim copies of the Program's source code as
+you receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice and
+disclaimer of warranty; keep intact all the notices that refer to this License
+and to the absence of any warranty; and give any other recipients of the
+Program a copy of this License along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and you may
+at your option offer warranty protection in exchange for a fee.
+
+2. You may modify your copy or copies of the Program or any portion of it, thus
+forming a work based on the Program, and copy and distribute such modifications
+or work under the terms of Section 1 above, provided that you also meet all of
+these conditions:
+
+ a) You must cause the modified files to carry prominent notices stating
+ that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in whole or
+ in part contains or is derived from the Program or any part thereof, to be
+ licensed as a whole at no charge to all third parties under the terms of
+ this License.
+
+ c) If the modified program normally reads commands interactively when run,
+ you must cause it, when started running for such interactive use in the
+ most ordinary way, to print or display an announcement including an
+ appropriate copyright notice and a notice that there is no warranty (or
+ else, saying that you provide a warranty) and that users may redistribute
+ the program under these conditions, and telling the user how to view a copy
+ of this License. (Exception: if the Program itself is interactive but does
+ not normally print such an announcement, your work based on the Program is
+ not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If identifiable
+sections of that work are not derived from the Program, and can be reasonably
+considered independent and separate works in themselves, then this License, and
+its terms, do not apply to those sections when you distribute them as separate
+works. But when you distribute the same sections as part of a whole which is a
+work based on the Program, the distribution of the whole must be on the terms
+of this License, whose permissions for other licensees extend to the entire
+whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest your
+rights to work written entirely by you; rather, the intent is to exercise the
+right to control the distribution of derivative or collective works based on
+the Program.
+
+In addition, mere aggregation of another work not based on the Program with the
+Program (or with a work based on the Program) on a volume of a storage or
+distribution medium does not bring the other work under the scope of this
+License.
+
+3. You may copy and distribute the Program (or a work based on it, under
+Section 2) in object code or executable form under the terms of Sections 1 and
+2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable source
+ code, which must be distributed under the terms of Sections 1 and 2 above
+ on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three years, to
+ give any third party, for a charge no more than your cost of physically
+ performing source distribution, a complete machine-readable copy of the
+ corresponding source code, to be distributed under the terms of Sections 1
+ and 2 above on a medium customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer to
+ distribute corresponding source code. (This alternative is allowed only
+ for noncommercial distribution and only if you received the program in
+ object code or executable form with such an offer, in accord with
+ Subsection b above.)
+
+The source code for a work means the preferred form of the work for making
+modifications to it. For an executable work, complete source code means all
+the source code for all modules it contains, plus any associated interface
+definition files, plus the scripts used to control compilation and installation
+of the executable. However, as a special exception, the source code
+distributed need not include anything that is normally distributed (in either
+source or binary form) with the major components (compiler, kernel, and so on)
+of the operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the source
+code from the same place counts as distribution of the source code, even though
+third parties are not compelled to copy the source along with the object code.
+
+4. You may not copy, modify, sublicense, or distribute the Program except as
+expressly provided under this License. Any attempt otherwise to copy, modify,
+sublicense or distribute the Program is void, and will automatically terminate
+your rights under this License. However, parties who have received copies, or
+rights, from you under this License will not have their licenses terminated so
+long as such parties remain in full compliance.
+
+5. You are not required to accept this License, since you have not signed it.
+However, nothing else grants you permission to modify or distribute the Program
+or its derivative works. These actions are prohibited by law if you do not
+accept this License. Therefore, by modifying or distributing the Program (or
+any work based on the Program), you indicate your acceptance of this License to
+do so, and all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+6. Each time you redistribute the Program (or any work based on the Program),
+the recipient automatically receives a license from the original licensor to
+copy, distribute or modify the Program subject to these terms and conditions.
+You may not impose any further restrictions on the recipients' exercise of the
+rights granted herein. You are not responsible for enforcing compliance by
+third parties to this License.
+
+7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues), conditions
+are imposed on you (whether by court order, agreement or otherwise) that
+contradict the conditions of this License, they do not excuse you from the
+conditions of this License. If you cannot distribute so as to satisfy
+simultaneously your obligations under this License and any other pertinent
+obligations, then as a consequence you may not distribute the Program at all.
+For example, if a patent license would not permit royalty-free redistribution
+of the Program by all those who receive copies directly or indirectly through
+you, then the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply and
+the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any patents or
+other property right claims or to contest validity of any such claims; this
+section has the sole purpose of protecting the integrity of the free software
+distribution system, which is implemented by public license practices. Many
+people have made generous contributions to the wide range of software
+distributed through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing to
+distribute software through any other system and a licensee cannot impose that
+choice.
+
+This section is intended to make thoroughly clear what is believed to be a
+consequence of the rest of this License.
+
+8. If the distribution and/or use of the Program is restricted in certain
+countries either by patents or by copyrighted interfaces, the original
+copyright holder who places the Program under this License may add an explicit
+geographical distribution limitation excluding those countries, so that
+distribution is permitted only in or among countries not thus excluded. In
+such case, this License incorporates the limitation as if written in the body
+of this License.
+
+9. The Free Software Foundation may publish revised and/or new versions of the
+General Public License from time to time. Such new versions will be similar in
+spirit to the present version, but may differ in detail to address new problems
+or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any later
+version", you have the option of following the terms and conditions either of
+that version or of any later version published by the Free Software Foundation.
+If the Program does not specify a version number of this License, you may
+choose any version ever published by the Free Software Foundation.
+
+10. If you wish to incorporate parts of the Program into other free programs
+whose distribution conditions are different, write to the author to ask for
+permission. For software which is copyrighted by the Free Software Foundation,
+write to the Free Software Foundation; we sometimes make exceptions for this.
+Our decision will be guided by the two goals of preserving the free status of
+all derivatives of our free software and of promoting the sharing and reuse of
+software generally.
+
+NO WARRANTY
+
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR
+THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE
+STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE
+PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
+PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE,
+YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL
+ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE
+PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR
+INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA
+BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER
+OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+END OF TERMS AND CONDITIONS
+
+How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest possible
+use to the public, the best way to achieve this is to make it free software
+which everyone can redistribute and change under these terms.
+
+To do so, attach the following notices to the program. It is safest to attach
+them to the start of each source file to most effectively convey the exclusion
+of warranty; and each file should have at least the "copyright" line and a
+pointer to where the full notice is found.
+
+ One line to give the program's name and a brief idea of what it does.
+
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by the Free
+ Software Foundation; either version 2 of the License, or (at your option)
+ any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc., 59
+ Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this when it
+starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author Gnomovision comes
+ with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free
+ software, and you are welcome to redistribute it under certain conditions;
+ type 'show c' for details.
+
+The hypothetical commands 'show w' and 'show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may be
+called something other than 'show w' and 'show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your school,
+if any, to sign a "copyright disclaimer" for the program, if necessary. Here
+is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ 'Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ signature of Ty Coon, 1 April 1989
+
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Library General Public
+License instead of this License.
+
+
+"CLASSPATH" EXCEPTION TO THE GPL
+
+Certain source files distributed by Oracle America and/or its affiliates are
+subject to the following clarification and special exception to the GPL, but
+only where Oracle has expressly included in the particular source file's header
+the words "Oracle designates this particular file as subject to the "Classpath"
+exception as provided by Oracle in the LICENSE file that accompanied this code."
+
+ Linking this library statically or dynamically with other modules is making
+ a combined work based on this library. Thus, the terms and conditions of
+ the GNU General Public License cover the whole combination.
+
+ As a special exception, the copyright holders of this library give you
+ permission to link this library with independent modules to produce an
+ executable, regardless of the license terms of these independent modules,
+ and to copy and distribute the resulting executable under terms of your
+ choice, provided that you also meet, for each linked independent module,
+ the terms and conditions of the license of that module. An independent
+ module is a module which is not derived from or based on this library. If
+ you modify this library, you may extend this exception to your version of
+ the library, but you are not obligated to do so. If you do not wish to do
+ so, delete this exception statement from your version.
+
+===========================================================================
+
+MIT License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+===========================================================================
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-common/pom.xml b/hyracks-fullstack/algebricks/algebricks-common/pom.xml
index e657319..033728e 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-common/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/pom.xml b/hyracks-fullstack/algebricks/algebricks-compiler/pom.xml
index f557d7a..5381b19 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/pom.xml b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
index 9c81540..0d8891c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
index 4466408..9c41cb9 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
@@ -46,6 +46,7 @@
SELECT,
SINK,
SPLIT,
+ SWITCH,
SUBPLAN,
TOKENIZE,
UNIONALL,
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
index 4a900af..c052d58 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
@@ -68,6 +68,7 @@
SORT_MERGE_EXCHANGE,
SPATIAL_JOIN,
SPLIT,
+ SWITCH,
STABLE_SORT,
STATS,
STREAM_LIMIT,
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
index cecbaf7..7f7f65f 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
@@ -130,7 +130,7 @@
@Override
public String toString() {
- return "NULL";
+ return "null";
}
@Override
@@ -167,7 +167,7 @@
@Override
public String toString() {
- return "MISSING";
+ return "missing";
}
@Override
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
index 6e1ccef..34ed225 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
@@ -38,6 +38,11 @@
return true;
}
+ default boolean isExternal() {
+ // A function is not external by default.
+ return false;
+ }
+
/**
* @param args,
* the arguments.
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index d350789..4596393 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -43,41 +43,42 @@
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public interface IMetadataProvider<S, I> {
- public IDataSource<S> findDataSource(S id) throws AlgebricksException;
+ IDataSource<S> findDataSource(S id) throws AlgebricksException;
/**
* Obs: A scanner may choose to contribute a null
* AlgebricksPartitionConstraint and implement
* contributeSchedulingConstraints instead.
*/
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException;
+ IProjectionFiltrationInfo<?> projectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo)
+ throws AlgebricksException;
- public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
- int[] printColumns, IPrinterFactory[] printerFactories, IAWriterFactory writerFactory,
- RecordDescriptor inputDesc) throws AlgebricksException;
+ Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink, int[] printColumns,
+ IPrinterFactory[] printerFactories, IAWriterFactory writerFactory, RecordDescriptor inputDesc)
+ throws AlgebricksException;
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
- int[] printColumns, IPrinterFactory[] printerFactories, IAWriterFactory writerFactory,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink, int[] printColumns,
+ IPrinterFactory[] printerFactories, IAWriterFactory writerFactory,
IResultSerializerFactoryProvider resultSerializerFactoryProvider, RecordDescriptor inputDesc,
IResultMetadata metadata, JobSpecification spec) throws AlgebricksException;
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> dataSource,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> dataSource,
IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
List<LogicalVariable> additionalNonKeyFields, JobGenContext context, JobSpecification jobSpec)
throws AlgebricksException;
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
LogicalVariable payLoadVar, List<LogicalVariable> additionalFilterKeyFields,
List<LogicalVariable> additionalNonFilteringFields, RecordDescriptor inputRecordDesc, JobGenContext context,
JobSpecification jobSpec, boolean bulkload) throws AlgebricksException;
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields,
List<LogicalVariable> additionalNonFilteringFields, RecordDescriptor inputRecordDesc, JobGenContext context,
@@ -87,42 +88,28 @@
* Creates the insert runtime of IndexInsertDeletePOperator, which models
* insert/delete operations into a secondary index.
*
- * @param dataSource
- * Target secondary index.
- * @param propagatedSchema
- * Output schema of the insert/delete operator to be created.
- * @param inputSchemas
- * Output schemas of the insert/delete operator to be created.
- * @param typeEnv
- * Type environment of the original IndexInsertDeleteOperator operator.
- * @param primaryKeys
- * Variables for the dataset's primary keys that the dataSource secondary index belongs to.
- * @param secondaryKeys
- * Variables for the secondary-index keys.
- * @param additionalNonKeyFields
- * Additional variables that can be passed to the secondary index as payload.
- * This can be useful when creating a second filter on a non-primary and non-secondary
- * fields for additional pruning power.
- * @param filterExpr
- * Filtering expression to be pushed inside the runtime op.
- * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
- * @param recordDesc
- * Output record descriptor of the runtime op to be created.
- * @param context
- * Job generation context.
- * @param spec
- * Target job specification.
- * @param secondaryKeysPipelines
- * Nested plans to extract secondary keys.
- * @param pipelineTopSchema
- * Schema of the primary pipeline for secondary keys.
- * @return
- * A Hyracks IOperatorDescriptor and its partition constraint.
+ * @param dataSource Target secondary index.
+ * @param propagatedSchema Output schema of the insert/delete operator to be created.
+ * @param inputSchemas Output schemas of the insert/delete operator to be created.
+ * @param typeEnv Type environment of the original IndexInsertDeleteOperator operator.
+ * @param primaryKeys Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+ * @param secondaryKeys Variables for the secondary-index keys.
+ * @param additionalNonKeyFields Additional variables that can be passed to the secondary index as payload.
+ * This can be useful when creating a second filter on a non-primary and non-secondary
+ * fields for additional pruning power.
+ * @param filterExpr Filtering expression to be pushed inside the runtime op.
+ * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+ * @param recordDesc Output record descriptor of the runtime op to be created.
+ * @param context Job generation context.
+ * @param spec Target job specification.
+ * @param secondaryKeysPipelines Nested plans to extract secondary keys.
+ * @param pipelineTopSchema Schema of the primary pipeline for secondary keys.
+ * @return A Hyracks IOperatorDescriptor and its partition constraint.
* @throws AlgebricksException
*/
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
- IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
- IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(IDataSourceIndex<I, S> dataSource,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv,
+ List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
JobGenContext context, JobSpecification spec, boolean bulkload,
List<List<AlgebricksPipeline>> secondaryKeysPipelines, IOperatorSchema pipelineTopSchema)
@@ -132,42 +119,28 @@
* Creates the delete runtime of IndexInsertDeletePOperator, which models
* insert/delete operations into a secondary index.
*
- * @param dataSource
- * Target secondary index.
- * @param propagatedSchema
- * Output schema of the insert/delete operator to be created.
- * @param inputSchemas
- * Output schemas of the insert/delete operator to be created.
- * @param typeEnv
- * Type environment of the original IndexInsertDeleteOperator operator.
- * @param primaryKeys
- * Variables for the dataset's primary keys that the dataSource secondary index belongs to.
- * @param secondaryKeys
- * Variables for the secondary-index keys.
- * @param additionalNonKeyFields
- * Additional variables that can be passed to the secondary index as payload.
- * This can be useful when creating a second filter on a non-primary and non-secondary
- * fields for additional pruning power.
- * @param filterExpr
- * Filtering expression to be pushed inside the runtime op.
- * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
- * @param recordDesc
- * Output record descriptor of the runtime op to be created.
- * @param context
- * Job generation context.
- * @param spec
- * Target job specification.
- * @param secondaryKeysPipelines
- * Nested plan to extract secondary keys.
- * @param pipelineTopSchema
- * Schema of the primary pipeline for secondary keys.
- * @return
- * A Hyracks IOperatorDescriptor and its partition constraint.
+ * @param dataSource Target secondary index.
+ * @param propagatedSchema Output schema of the insert/delete operator to be created.
+ * @param inputSchemas Output schemas of the insert/delete operator to be created.
+ * @param typeEnv Type environment of the original IndexInsertDeleteOperator operator.
+ * @param primaryKeys Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+ * @param secondaryKeys Variables for the secondary-index keys.
+ * @param additionalNonKeyFields Additional variables that can be passed to the secondary index as payload.
+ * This can be useful when creating a second filter on a non-primary and non-secondary
+ * fields for additional pruning power.
+ * @param filterExpr Filtering expression to be pushed inside the runtime op.
+ * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+ * @param recordDesc Output record descriptor of the runtime op to be created.
+ * @param context Job generation context.
+ * @param spec Target job specification.
+ * @param secondaryKeysPipelines Nested plan to extract secondary keys.
+ * @param pipelineTopSchema Schema of the primary pipeline for secondary keys.
+ * @return A Hyracks IOperatorDescriptor and its partition constraint.
* @throws AlgebricksException
*/
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
- IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
- IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(IDataSourceIndex<I, S> dataSource,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv,
+ List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
JobGenContext context, JobSpecification spec, List<List<AlgebricksPipeline>> secondaryKeysPipelines,
IOperatorSchema pipelineTopSchema) throws AlgebricksException;
@@ -177,48 +150,37 @@
* secondary key into [token, number of token] pair in a length-partitioned index.
* In case of non length-partitioned index, it tokenizes secondary key into [token].
*
- * @param dataSource
- * Target secondary index.
- * @param propagatedSchema
- * Output schema of the insert/delete operator to be created.
- * @param inputSchemas
- * Output schemas of the insert/delete operator to be created.
- * @param typeEnv
- * Type environment of the original IndexInsertDeleteOperator operator.
- * @param primaryKeys
- * Variables for the dataset's primary keys that the dataSource secondary index belongs to.
- * @param secondaryKeys
- * Variables for the secondary-index keys.
- * @param filterExpr
- * Filtering expression to be pushed inside the runtime op.
- * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
- * @param recordDesc
- * Output record descriptor of the runtime op to be created.
- * @param context
- * Job generation context.
- * @param spec
- * Target job specification.
- * @return
- * A Hyracks IOperatorDescriptor and its partition constraint.
+ * @param dataSource Target secondary index.
+ * @param propagatedSchema Output schema of the insert/delete operator to be created.
+ * @param inputSchemas Output schemas of the insert/delete operator to be created.
+ * @param typeEnv Type environment of the original IndexInsertDeleteOperator operator.
+ * @param primaryKeys Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+ * @param secondaryKeys Variables for the secondary-index keys.
+ * @param filterExpr Filtering expression to be pushed inside the runtime op.
+ * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+ * @param recordDesc Output record descriptor of the runtime op to be created.
+ * @param context Job generation context.
+ * @param spec Target job specification.
+ * @return A Hyracks IOperatorDescriptor and its partition constraint.
* @throws AlgebricksException
*/
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getTokenizerRuntime(
- IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
- IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec,
- boolean bulkload) throws AlgebricksException;
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getTokenizerRuntime(IDataSourceIndex<I, S> dataSource,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv,
+ List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys, ILogicalExpression filterExpr,
+ RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, boolean bulkload)
+ throws AlgebricksException;
- public IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
+ IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
- public IFunctionInfo lookupFunction(FunctionIdentifier fid);
+ IFunctionInfo lookupFunction(FunctionIdentifier fid);
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getUpsertRuntime(IDataSource<S> dataSource,
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getUpsertRuntime(IDataSource<S> dataSource,
IOperatorSchema inputSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
LogicalVariable payLoadVar, List<LogicalVariable> additionalFilterFields,
List<LogicalVariable> additionalNonFilteringFields, RecordDescriptor recordDesc, JobGenContext context,
JobSpecification jobSpec) throws AlgebricksException;
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexUpsertRuntime(
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexUpsertRuntime(
IDataSourceIndex<I, S> dataSourceIndex, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalFilteringKeys, ILogicalExpression filterExpr,
@@ -226,12 +188,11 @@
LogicalVariable prevAdditionalFilteringKeys, RecordDescriptor inputDesc, JobGenContext context,
JobSpecification spec, List<List<AlgebricksPipeline>> secondaryKeysPipelines) throws AlgebricksException;
- public ITupleFilterFactory createTupleFilterFactory(IOperatorSchema[] inputSchemas,
- IVariableTypeEnvironment typeEnv, ILogicalExpression filterExpr, JobGenContext context)
- throws AlgebricksException;
+ ITupleFilterFactory createTupleFilterFactory(IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv,
+ ILogicalExpression filterExpr, JobGenContext context) throws AlgebricksException;
- public Map<String, Object> getConfig();
+ Map<String, Object> getConfig();
- public boolean isBlockingOperatorDisabled();
+ boolean isBlockingOperatorDisabled();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionFiltrationInfo.java
similarity index 80%
rename from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
rename to hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionFiltrationInfo.java
index 3c1a24d..9973d08 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionFiltrationInfo.java
@@ -18,18 +18,22 @@
*/
package org.apache.hyracks.algebricks.core.algebra.metadata;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+
/**
* Generic interface to include the projection information for
* {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
*/
-public interface IProjectionInfo<T> {
+public interface IProjectionFiltrationInfo<T> {
/**
* @return projected values' information
*/
T getProjectionInfo();
+ ILogicalExpression getFilterExpression();
+
/**
- * @return a copy of the {@link IProjectionInfo}
+ * @return a copy of the {@link IProjectionFiltrationInfo}
*/
- IProjectionInfo<T> createCopy();
+ IProjectionFiltrationInfo<T> createCopy();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
index 5fc69b2..56e2e22 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
@@ -25,10 +25,10 @@
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
public abstract class AbstractUnnestMapOperator extends AbstractUnnestOperator {
- protected final Mutable<ILogicalExpression> expression;
protected final List<Object> variableTypes;
protected boolean propagateInput;
protected List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
@@ -42,7 +42,6 @@
public AbstractUnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
List<Object> variableTypes, boolean propagateInput) {
super(variables, expression);
- this.expression = expression;
this.variableTypes = variableTypes;
this.propagateInput = propagateInput;
this.propagateIndexFilter = false;
@@ -90,6 +89,18 @@
};
}
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean changed = super.acceptExpressionTransform(visitor);
+
+ if (additionalFilteringExpressions != null) {
+ for (Mutable<ILogicalExpression> filterExpr : additionalFilteringExpressions) {
+ changed |= visitor.transform(filterExpr);
+ }
+ }
+ return changed;
+ }
+
public boolean propagatesInput() {
return propagateInput;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
index bb18014..e3ce82d 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
@@ -29,7 +29,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
@@ -50,19 +50,22 @@
// the maximum of number of results output by this operator
private long outputLimit = -1;
- private IProjectionInfo<?> projectionInfo;
+ private IProjectionFiltrationInfo<?> datasetProjectionInfo;
+ private IProjectionFiltrationInfo<?> metaProjectionInfo;
public DataSourceScanOperator(List<LogicalVariable> variables, IDataSource<?> dataSource) {
- this(variables, dataSource, null, -1, null);
+ this(variables, dataSource, null, -1, null, null);
}
public DataSourceScanOperator(List<LogicalVariable> variables, IDataSource<?> dataSource,
- Mutable<ILogicalExpression> selectCondition, long outputLimit, IProjectionInfo projectionInfo) {
+ Mutable<ILogicalExpression> selectCondition, long outputLimit,
+ IProjectionFiltrationInfo<?> datasetProjectionInfo, IProjectionFiltrationInfo<?> metaProjectionInfo) {
super(variables, dataSource);
projectVars = new ArrayList<>();
this.selectCondition = selectCondition;
this.outputLimit = outputLimit;
- this.projectionInfo = projectionInfo;
+ this.datasetProjectionInfo = datasetProjectionInfo;
+ this.metaProjectionInfo = metaProjectionInfo;
}
@Override
@@ -77,7 +80,13 @@
@Override
public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
- return false;
+ boolean changed = selectCondition != null && visitor.transform(selectCondition);
+ if (additionalFilteringExpressions != null) {
+ for (Mutable<ILogicalExpression> filterExpr : additionalFilteringExpressions) {
+ changed |= visitor.transform(filterExpr);
+ }
+ }
+ return changed;
}
@Override
@@ -167,11 +176,19 @@
this.outputLimit = outputLimit;
}
- public void setProjectionInfo(IProjectionInfo<?> projectionInfo) {
- this.projectionInfo = projectionInfo;
+ public void setDatasetProjectionInfo(IProjectionFiltrationInfo<?> datasetProjectionInfo) {
+ this.datasetProjectionInfo = datasetProjectionInfo;
}
- public IProjectionInfo<?> getProjectionInfo() {
- return projectionInfo;
+ public IProjectionFiltrationInfo<?> getDatasetProjectionInfo() {
+ return datasetProjectionInfo;
+ }
+
+ public void setMetaProjectionInfo(IProjectionFiltrationInfo<?> metaProjectionInfo) {
+ this.metaProjectionInfo = metaProjectionInfo;
+ }
+
+ public IProjectionFiltrationInfo<?> getMetaProjectionInfo() {
+ return metaProjectionInfo;
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SwitchOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SwitchOperator.java
new file mode 100644
index 0000000..4b1f260
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SwitchOperator.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+/**
+ * Switch Operator receives an expression and an output mapping. We evaluate the expression during runtime and look up
+ * the result in the output mapping. Based on this, we propagate each tuple to the corresponding output branch(es).
+ */
+public class SwitchOperator extends AbstractReplicateOperator {
+
+ // Expression containing the index of the relevant field
+ private final Mutable<ILogicalExpression> branchingExpression;
+
+ // The supplied mapping from field values to arrays of output branch numbers
+ private final Map<Integer, int[]> outputMapping;
+
+ public SwitchOperator(int outputArity, Mutable<ILogicalExpression> branchingExpression,
+ Map<Integer, int[]> outputMapping) {
+ super(outputArity);
+ this.branchingExpression = branchingExpression;
+ this.outputMapping = outputMapping;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.SWITCH;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitSwitchOperator(this, arg);
+ }
+
+ public Mutable<ILogicalExpression> getBranchingExpression() {
+ return branchingExpression;
+ }
+
+ public Map<Integer, int[]> getOutputMapping() {
+ return outputMapping;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ return visitor.transform(branchingExpression);
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
index c4bcc52..6d11931 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -26,8 +26,10 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext;
import org.apache.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
public class UnnestMapOperator extends AbstractUnnestMapOperator {
@@ -38,14 +40,18 @@
// the maximum of number of results output by this operator
private long outputLimit = -1;
+ private IProjectionFiltrationInfo<?> datasetProjectionInfo;
+ private IProjectionFiltrationInfo<?> metaProjectionInfo;
+
public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
List<Object> variableTypes, boolean propagateInput) {
- this(variables, expression, variableTypes, propagateInput, null, -1);
+ this(variables, expression, variableTypes, propagateInput, null, -1, null, null);
}
public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
List<Object> variableTypes, boolean propagateInput, Mutable<ILogicalExpression> selectCondition,
- long outputLimit) {
+ long outputLimit, IProjectionFiltrationInfo<?> datasetProjectionInfo,
+ IProjectionFiltrationInfo<?> metaProjectionInfo) {
super(variables, expression, variableTypes, propagateInput);
this.selectCondition = selectCondition;
this.outputLimit = outputLimit;
@@ -61,6 +67,12 @@
return visitor.visitUnnestMapOperator(this, arg);
}
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean changed = super.acceptExpressionTransform(visitor);
+ return selectCondition != null && visitor.transform(selectCondition) || changed;
+ }
+
// When propagateInput is true,
// this operator propagates all input variables.
@Override
@@ -94,4 +106,20 @@
this.outputLimit = outputLimit;
}
+ public void setDatasetProjectionInfo(IProjectionFiltrationInfo<?> projectionInfo) {
+ this.datasetProjectionInfo = projectionInfo;
+ }
+
+ public IProjectionFiltrationInfo<?> getDatasetProjectionInfo() {
+ return datasetProjectionInfo;
+ }
+
+ public void setMetaProjectionInfo(IProjectionFiltrationInfo<?> metaProjectionInfo) {
+ this.metaProjectionInfo = metaProjectionInfo;
+ }
+
+ public IProjectionFiltrationInfo<?> getMetaProjectionInfo() {
+ return metaProjectionInfo;
+ }
+
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java
index 6ed90a5..14221d6 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java
@@ -58,6 +58,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -196,6 +197,11 @@
}
@Override
+ public Long visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return op.getInputs().get(0).getValue().accept(this, arg);
+ }
+
+ @Override
public Long visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return op.getInputs().get(0).getValue().accept(this, arg);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
index f3717c8..5937d9d 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
@@ -74,6 +74,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -427,6 +428,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, IOptimizationContext ctx) throws AlgebricksException {
propagateFDsAndEquivClasses(op, ctx);
return null;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index 9e2e87c..5a6574a 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -27,6 +27,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -66,6 +67,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -310,6 +312,12 @@
}
@Override
+ public Boolean visitSwitchOperator(SwitchOperator op, ILogicalOperator arg) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public Boolean visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg) throws AlgebricksException {
AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
if (aop.getOperatorTag() != LogicalOperatorTag.MATERIALIZE) {
@@ -439,7 +447,9 @@
if (!isomorphic) {
return Boolean.FALSE;
}
- isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
+ isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue())
+ && Objects.equals(op.getDatasetProjectionInfo(), unnestOpArg.getDatasetProjectionInfo())
+ && Objects.equals(op.getMetaProjectionInfo(), unnestOpArg.getMetaProjectionInfo());
return isomorphic;
}
@@ -472,7 +482,8 @@
DataSourceScanOperator argScan = (DataSourceScanOperator) arg;
boolean isomorphic = op.getDataSource().getId().equals(argScan.getDataSource().getId())
&& op.getOutputLimit() == argScan.getOutputLimit()
- && Objects.equals(op.getProjectionInfo(), argScan.getProjectionInfo());
+ && Objects.equals(op.getDatasetProjectionInfo(), argScan.getDatasetProjectionInfo())
+ && Objects.equals(op.getMetaProjectionInfo(), argScan.getMetaProjectionInfo());
if (!isomorphic) {
return Boolean.FALSE;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
index 8ca2b83..e4d6586 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
@@ -65,6 +65,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -205,6 +206,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg) throws AlgebricksException {
mapVariablesStandard(op, arg);
return null;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java
index e242531..71a659b 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java
@@ -28,6 +28,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -36,7 +37,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.IVariableContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
@@ -64,6 +65,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -102,22 +104,17 @@
private final boolean reuseFreeVars;
/**
- * @param varContext
- * , the variable context.
- * @param typeContext
- * the type context.
+ * @param varContext , the variable context.
+ * @param typeContext the type context.
*/
public LogicalOperatorDeepCopyWithNewVariablesVisitor(IVariableContext varContext, ITypingContext typeContext) {
this(varContext, typeContext, new LinkedHashMap<>(), false);
}
/**
- * @param varContext
- * , the variable context.
- * @param typeContext
- * the type context.
- * @param reuseFreeVars
- * whether free variables in the given plan tree should be reused.
+ * @param varContext , the variable context.
+ * @param typeContext the type context.
+ * @param reuseFreeVars whether free variables in the given plan tree should be reused.
*/
public LogicalOperatorDeepCopyWithNewVariablesVisitor(IVariableContext varContext, ITypingContext typeContext,
boolean reuseFreeVars) {
@@ -125,16 +122,12 @@
}
/**
- * @param varContext
- * , the variable context.
- * @param typeContext
- * the type context.
- * @param inVarMapping
- * Variable mapping keyed by variables in the original plan.
- * Those variables are replaced by their corresponding value in
- * the map in the copied plan.
- * @param reuseFreeVars
- * whether free variables in the given plan tree should be reused.
+ * @param varContext , the variable context.
+ * @param typeContext the type context.
+ * @param inVarMapping Variable mapping keyed by variables in the original plan.
+ * Those variables are replaced by their corresponding value in
+ * the map in the copied plan.
+ * @param reuseFreeVars whether free variables in the given plan tree should be reused.
*/
public LogicalOperatorDeepCopyWithNewVariablesVisitor(IVariableContext varContext, ITypingContext typeContext,
LinkedHashMap<LogicalVariable, LogicalVariable> inVarMapping, boolean reuseFreeVars) {
@@ -324,9 +317,12 @@
throws AlgebricksException {
Mutable<ILogicalExpression> newSelectCondition = op.getSelectCondition() != null
? exprDeepCopyVisitor.deepCopyExpressionReference(op.getSelectCondition()) : null;
- IProjectionInfo<?> projectionInfo = op.getProjectionInfo() != null ? op.getProjectionInfo().createCopy() : null;
+ IProjectionFiltrationInfo<?> datasetProjectionInfo =
+ op.getDatasetProjectionInfo() != null ? op.getDatasetProjectionInfo().createCopy() : null;
+ IProjectionFiltrationInfo<?> metaProjectionInfo =
+ op.getMetaProjectionInfo() != null ? op.getMetaProjectionInfo().createCopy() : null;
DataSourceScanOperator opCopy = new DataSourceScanOperator(deepCopyVariableList(op.getVariables()),
- op.getDataSource(), newSelectCondition, op.getOutputLimit(), projectionInfo);
+ op.getDataSource(), newSelectCondition, op.getOutputLimit(), datasetProjectionInfo, metaProjectionInfo);
deepCopyInputsAnnotationsAndExecutionMode(op, arg, opCopy);
return opCopy;
}
@@ -447,6 +443,12 @@
}
@Override
+ public ILogicalOperator visitSwitchOperator(SwitchOperator op, ILogicalOperator arg) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public ILogicalOperator visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg)
throws AlgebricksException {
MaterializeOperator opCopy = new MaterializeOperator();
@@ -532,9 +534,14 @@
throws AlgebricksException {
Mutable<ILogicalExpression> newSelectCondition = op.getSelectCondition() != null
? exprDeepCopyVisitor.deepCopyExpressionReference(op.getSelectCondition()) : null;
+ IProjectionFiltrationInfo<?> datasetProjectionInfo =
+ op.getDatasetProjectionInfo() != null ? op.getDatasetProjectionInfo().createCopy() : null;
+ IProjectionFiltrationInfo<?> metaProjectionInfo =
+ op.getMetaProjectionInfo() != null ? op.getMetaProjectionInfo().createCopy() : null;
UnnestMapOperator opCopy = new UnnestMapOperator(deepCopyVariableList(op.getVariables()),
exprDeepCopyVisitor.deepCopyExpressionReference(op.getExpressionRef()), op.getVariableTypes(),
- op.propagatesInput(), newSelectCondition, op.getOutputLimit());
+ op.propagatesInput(), newSelectCondition, op.getOutputLimit(), datasetProjectionInfo,
+ metaProjectionInfo);
deepCopyInputsAnnotationsAndExecutionMode(op, arg, opCopy);
return opCopy;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
index 5a566ee..3a88d2c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
@@ -57,6 +57,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -194,6 +195,11 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, IOptimizationContext arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, IOptimizationContext arg) throws AlgebricksException {
// TODO Auto-generated method stub
return null;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
index c2ee661..b7029d1 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import org.apache.commons.lang3.mutable.Mutable;
@@ -32,7 +33,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
@@ -63,6 +64,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -176,7 +178,13 @@
@Override
public ILogicalOperator visitSplitOperator(SplitOperator op, Void arg) throws AlgebricksException {
- return new SplitOperator(op.getOutputArity(), op.getBranchingExpression());
+ return new SplitOperator(op.getOutputArity(), deepCopyExpressionRef(op.getBranchingExpression()));
+ }
+
+ @Override
+ public ILogicalOperator visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return new SwitchOperator(op.getOutputArity(), deepCopyExpressionRef(op.getBranchingExpression()),
+ new HashMap<>(op.getOutputMapping()));
}
@Override
@@ -237,8 +245,13 @@
newInputList.addAll(op.getVariables());
Mutable<ILogicalExpression> newSelectCondition =
op.getSelectCondition() != null ? deepCopyExpressionRef(op.getSelectCondition()) : null;
+ IProjectionFiltrationInfo<?> datasetProjectionInfo =
+ op.getDatasetProjectionInfo() != null ? op.getDatasetProjectionInfo().createCopy() : null;
+ IProjectionFiltrationInfo<?> metaProjectionInfo =
+ op.getMetaProjectionInfo() != null ? op.getMetaProjectionInfo().createCopy() : null;
return new UnnestMapOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()),
- new ArrayList<>(op.getVariableTypes()), op.propagatesInput(), newSelectCondition, op.getOutputLimit());
+ new ArrayList<>(op.getVariableTypes()), op.propagatesInput(), newSelectCondition, op.getOutputLimit(),
+ datasetProjectionInfo, metaProjectionInfo);
}
@Override
@@ -256,10 +269,13 @@
newInputList.addAll(op.getVariables());
Mutable<ILogicalExpression> newSelectCondition =
op.getSelectCondition() != null ? deepCopyExpressionRef(op.getSelectCondition()) : null;
- IProjectionInfo<?> projectionInfo = op.getProjectionInfo() != null ? op.getProjectionInfo().createCopy() : null;
+ IProjectionFiltrationInfo<?> datasetProjectionInfo =
+ op.getDatasetProjectionInfo() != null ? op.getDatasetProjectionInfo().createCopy() : null;
+ IProjectionFiltrationInfo<?> metaProjectionInfo =
+ op.getMetaProjectionInfo() != null ? op.getMetaProjectionInfo().createCopy() : null;
return new DataSourceScanOperator(newInputList, op.getDataSource(), newSelectCondition, op.getOutputLimit(),
- projectionInfo);
+ datasetProjectionInfo, metaProjectionInfo);
}
@Override
@@ -371,7 +387,7 @@
private void deepCopyExpressionRefs(List<Mutable<ILogicalExpression>> newExprs,
List<Mutable<ILogicalExpression>> oldExprs) {
for (Mutable<ILogicalExpression> oldExpr : oldExprs) {
- newExprs.add(new MutableObject<>(((AbstractLogicalExpression) oldExpr.getValue()).cloneExpression()));
+ newExprs.add(new MutableObject<>(oldExpr.getValue().cloneExpression()));
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/PrimaryKeyVariablesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/PrimaryKeyVariablesVisitor.java
index eb90288..8a3a885 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/PrimaryKeyVariablesVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/PrimaryKeyVariablesVisitor.java
@@ -58,6 +58,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -186,6 +187,11 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, IOptimizationContext arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, IOptimizationContext ctx) throws AlgebricksException {
return null;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
index 5d9d7895..ff50994 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
@@ -62,6 +62,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -271,6 +272,11 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return null;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
index 44bb7e2..e7d6a92 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
@@ -60,6 +60,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -290,6 +291,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
standardLayout(op);
return null;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
index 439e493..cf8196c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -23,6 +23,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -63,6 +64,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -410,6 +412,13 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Pair<LogicalVariable, LogicalVariable> pair)
throws AlgebricksException {
// does not produce/use any variables
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
index 174b184..4c994b5 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -25,6 +25,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -64,6 +65,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -473,6 +475,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return null;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
index 48dc607..866334a 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -116,10 +116,10 @@
scan.getSelectCondition().getValue(), context);
}
- Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p =
- mp.getScannerRuntime(dataSource, vars, projectVars, scan.isProjectPushed(), scan.getMinFilterVars(),
- scan.getMaxFilterVars(), tupleFilterFactory, scan.getOutputLimit(), opSchema, typeEnv, context,
- builder.getJobSpec(), implConfig, scan.getProjectionInfo());
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = mp.getScannerRuntime(dataSource, vars, projectVars,
+ scan.isProjectPushed(), scan.getMinFilterVars(), scan.getMaxFilterVars(), tupleFilterFactory,
+ scan.getOutputLimit(), opSchema, typeEnv, context, builder.getJobSpec(), implConfig,
+ scan.getDatasetProjectionInfo(), scan.getMetaProjectionInfo());
IOperatorDescriptor opDesc = p.first;
opDesc.setSourceLocation(scan.getSourceLocation());
builder.contributeHyracksOperator(scan, opDesc);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
index 083e4d3..2bd78b1 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
@@ -27,7 +27,6 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import org.apache.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.job.IConnectorDescriptorRegistry;
@@ -43,7 +42,9 @@
@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
- deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ if (op2.getDeliveredPhysicalProperties() != null) {
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+ }
}
@Override
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SwitchPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SwitchPOperator.java
new file mode 100644
index 0000000..4b58788
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SwitchPOperator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.Map;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import org.apache.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.operators.std.SwitchOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+
+public class SwitchPOperator extends AbstractReplicatePOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.SWITCH;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ SwitchOperator sop = (SwitchOperator) op;
+ int outputArity = sop.getOutputArity();
+ Map<Integer, int[]> outputMapping = sop.getOutputMapping();
+
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
+ RecordDescriptor recDescriptor =
+ JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
+
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+ IScalarEvaluatorFactory branchingExprEvalFactory = expressionRuntimeProvider.createEvaluatorFactory(
+ sop.getBranchingExpression().getValue(), context.getTypeEnvironment(op), inputSchemas, context);
+
+ IBinaryIntegerInspectorFactory intInspectorFactory = context.getBinaryIntegerInspectorFactory();
+
+ SwitchOperatorDescriptor sopDesc = new SwitchOperatorDescriptor(spec, recDescriptor, outputArity,
+ branchingExprEvalFactory, intInspectorFactory, outputMapping);
+ sopDesc.setSourceLocation(sop.getSourceLocation());
+
+ contributeOpDesc(builder, sop, sopDesc);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/IPlanPrettyPrinter.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/IPlanPrettyPrinter.java
index 66d48d3..2b53de4 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/IPlanPrettyPrinter.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/IPlanPrettyPrinter.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.algebricks.core.algebra.prettyprint;
+import java.util.Map;
+
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -43,6 +45,10 @@
/** Prints the whole logical plan. */
IPlanPrettyPrinter printPlan(ILogicalPlan plan, boolean printOptimizerEstimates) throws AlgebricksException;
+ /** Prints the logical plan, annotated with physical operator and connector ids */
+ IPlanPrettyPrinter printPlan(ILogicalPlan plan, Map<Object, String> log2phys, boolean printOptimizerEstimates)
+ throws AlgebricksException;
+
/** Resets the state of the pretty printer. */
IPlanPrettyPrinter reset() throws AlgebricksException;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
index a60c969..069012b 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -20,9 +20,11 @@
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -31,7 +33,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
@@ -65,6 +67,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -97,6 +100,14 @@
}
@Override
+ public final IPlanPrettyPrinter printPlan(ILogicalPlan plan, Map<Object, String> log2phys,
+ boolean printOptimizerEstimates) throws AlgebricksException {
+ //TODO(ian): would be nice if the text plan returned real operator ids too
+ printPlanImpl(plan, 0, printOptimizerEstimates);
+ return this;
+ }
+
+ @Override
public final IPlanPrettyPrinter printOperator(AbstractLogicalOperator op, boolean printInputs,
boolean printOptimizerEstimates) throws AlgebricksException {
printOperatorImpl(op, 0, printInputs, printOptimizerEstimates);
@@ -348,6 +359,9 @@
AlgebricksStringBuilderWriter plan = printAbstractUnnestMapOperator(op, indent, "unnest-map", null);
appendSelectConditionInformation(plan, op.getSelectCondition(), indent);
appendLimitInformation(plan, op.getOutputLimit());
+ appendProjectInformation(plan, "project", op.getDatasetProjectionInfo());
+ appendProjectInformation(plan, "project-meta", op.getMetaProjectionInfo());
+ appendFilterExpression(plan, op.getDatasetProjectionInfo());
return null;
}
@@ -375,7 +389,9 @@
appendFilterInformation(plan, op.getMinFilterVars(), op.getMaxFilterVars());
appendSelectConditionInformation(plan, op.getSelectCondition(), indent);
appendLimitInformation(plan, op.getOutputLimit());
- appendProjectInformation(plan, op.getProjectionInfo());
+ appendProjectInformation(plan, "project", op.getDatasetProjectionInfo());
+ appendProjectInformation(plan, "project-meta", op.getMetaProjectionInfo());
+ appendFilterExpression(plan, op.getDatasetProjectionInfo());
return null;
}
@@ -406,15 +422,30 @@
}
}
- private void appendProjectInformation(AlgebricksStringBuilderWriter plan, IProjectionInfo<?> projectionInfo) {
+ private void appendProjectInformation(AlgebricksStringBuilderWriter plan, String projectionSource,
+ IProjectionFiltrationInfo<?> projectionInfo) {
final String projectedFields = projectionInfo == null ? "" : projectionInfo.toString();
if (!projectedFields.isEmpty()) {
- plan.append(" project (");
+ plan.append(" ");
+ plan.append(projectionSource);
+ plan.append(" (");
plan.append(projectedFields);
plan.append(")");
}
}
+ private void appendFilterExpression(AlgebricksStringBuilderWriter plan,
+ IProjectionFiltrationInfo<?> projectionInfo) {
+ final String filterExpr = projectionInfo == null || projectionInfo.getFilterExpression() == null ? ""
+ : projectionInfo.getFilterExpression().toString();
+ if (!filterExpr.isEmpty()) {
+ plan.append(" filter on ");
+ plan.append("(");
+ plan.append(filterExpr);
+ plan.append(")");
+ }
+ }
+
@Override
public Void visitLimitOperator(LimitOperator op, Integer indent) throws AlgebricksException {
addIndent(indent).append("limit");
@@ -453,6 +484,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Integer indent) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Integer indent) throws AlgebricksException {
addIndent(indent).append("materialize");
return null;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
index 464d15e..115448e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.algebricks.core.algebra.prettyprint;
import java.io.IOException;
+import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedList;
@@ -27,6 +28,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -36,7 +38,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
@@ -72,6 +74,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -100,6 +103,7 @@
private static final String MISSING_VALUE_FIELD = "missing-value";
private static final String OPTIMIZER_ESTIMATES = "optimizer-estimates";
private final Map<AbstractLogicalOperator, String> operatorIdentity = new HashMap<>();
+ private Map<Object, String> log2odid = Collections.emptyMap();
private final IdCounter idCounter = new IdCounter();
private final JsonGenerator jsonGenerator;
@@ -164,7 +168,15 @@
}
@Override
+ public final IPlanPrettyPrinter printPlan(ILogicalPlan plan, Map<Object, String> log2phys,
+ boolean printOptimizerEstimates) throws AlgebricksException {
+ this.log2odid = log2phys;
+ printPlanImpl(plan, printOptimizerEstimates);
+ flushContentToWriter();
+ return this;
+ }
+ @Override
public final IPlanPrettyPrinter printOperator(AbstractLogicalOperator op, boolean printInputs,
boolean printOptimizerEstimates) throws AlgebricksException {
printOperatorImpl(op, printInputs, printOptimizerEstimates);
@@ -195,6 +207,10 @@
jsonGenerator.writeStartObject();
op.accept(this, null);
jsonGenerator.writeStringField("operatorId", idCounter.printOperatorId(op));
+ String od = log2odid.get(op);
+ if (od != null) {
+ jsonGenerator.writeStringField("runtime-id", od);
+ }
IPhysicalOperator pOp = op.getPhysicalOperator();
if (pOp != null) {
jsonGenerator.writeStringField("physical-operator", pOp.toString(false));
@@ -529,6 +545,9 @@
try {
writeUnnestMapOperator(op, indent, "unnest-map", null);
writeSelectLimitInformation(op.getSelectCondition(), op.getOutputLimit(), indent);
+ writeProjectInformation("project", op.getDatasetProjectionInfo());
+ writeProjectInformation("project-meta", op.getMetaProjectionInfo());
+ writeFilterInformation(op.getDatasetProjectionInfo());
return null;
} catch (IOException e) {
throw AlgebricksException.create(ErrorCode.ERROR_PRINTING_PLAN, e, String.valueOf(e));
@@ -558,7 +577,9 @@
}
writeFilterInformation(op.getMinFilterVars(), op.getMaxFilterVars());
writeSelectLimitInformation(op.getSelectCondition(), op.getOutputLimit(), indent);
- writeProjectInformation(op.getProjectionInfo());
+ writeProjectInformation("project", op.getDatasetProjectionInfo());
+ writeProjectInformation("project-meta", op.getMetaProjectionInfo());
+ writeFilterInformation(op.getDatasetProjectionInfo());
return null;
} catch (IOException e) {
throw AlgebricksException.create(ErrorCode.ERROR_PRINTING_PLAN, e, String.valueOf(e));
@@ -631,6 +652,12 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void indent) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void indent) throws AlgebricksException {
try {
jsonGenerator.writeStringField(OPERATOR_FIELD, "materialize");
@@ -881,10 +908,19 @@
}
}
- private void writeProjectInformation(IProjectionInfo<?> projectionInfo) throws IOException {
+ private void writeProjectInformation(String projectionSource, IProjectionFiltrationInfo<?> projectionInfo)
+ throws IOException {
final String projectedFields = projectionInfo == null ? "" : projectionInfo.toString();
if (!projectedFields.isEmpty()) {
- jsonGenerator.writeStringField("project", projectedFields);
+ jsonGenerator.writeStringField(projectionSource, projectedFields);
+ }
+ }
+
+ private void writeFilterInformation(IProjectionFiltrationInfo<?> projectionInfo) throws IOException {
+ final String filterExpr = projectionInfo == null || projectionInfo.getFilterExpression() == null ? ""
+ : projectionInfo.getFilterExpression().toString();
+ if (!filterExpr.isEmpty()) {
+ jsonGenerator.writeStringField("filter-on", filterExpr);
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
index d521831..4eb6494 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
@@ -48,6 +48,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -88,6 +89,8 @@
public R visitSplitOperator(SplitOperator op, T arg) throws AlgebricksException;
+ public R visitSwitchOperator(SwitchOperator op, T arg) throws AlgebricksException;
+
public R visitMaterializeOperator(MaterializeOperator op, T arg) throws AlgebricksException;
public R visitScriptOperator(ScriptOperator op, T arg) throws AlgebricksException;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/LogicalExpressionReferenceTransformVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/LogicalExpressionReferenceTransformVisitor.java
index 9350f95..f1613a5 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/LogicalExpressionReferenceTransformVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/LogicalExpressionReferenceTransformVisitor.java
@@ -50,6 +50,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -166,6 +167,12 @@
}
@Override
+ public Boolean visitSwitchOperator(SwitchOperator op, ILogicalExpressionReferenceTransform arg)
+ throws AlgebricksException {
+ return visitOperator(op, arg);
+ }
+
+ @Override
public Boolean visitMaterializeOperator(MaterializeOperator op, ILogicalExpressionReferenceTransform arg)
throws AlgebricksException {
return visitOperator(op, arg);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index e53322c..0d02203 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -44,4 +44,6 @@
public static final String QUERY_PLAN_SHAPE_DEFAULT = QUERY_PLAN_SHAPE_ZIGZAG;
public static final int EXTERNAL_SCAN_BUFFER_SIZE =
StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
+ public static final boolean BATCH_LOOKUP_DEFAULT = false;
+ public static final boolean COLUMN_FILTER_DEFAULT = false;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
index c8fff99..d8bf190 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
@@ -21,6 +21,7 @@
import static org.apache.hyracks.api.exceptions.ErrorCode.DESCRIPTOR_GENERATION_ERROR;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -147,6 +148,15 @@
hyracksOps.put(op, opDesc);
}
+ public Map<Object, String> getLogical2PhysicalMap() {
+ Map<ILogicalOperator, String> mergedOperatorMap = new HashMap<>();
+ hyracksOps.forEach(((k, v) -> mergedOperatorMap.put(k, v.getOperatorId().toString())));
+ algebraicOpBelongingToMetaAsterixOp
+ .forEach((k, v) -> mergedOperatorMap.put(k, metaAsterixOps.get(v).getOperatorId().toString()));
+ connectors.forEach((k, v) -> mergedOperatorMap.put(k, v.getFirst().getConnectorId().toString()));
+ return Collections.unmodifiableMap(mergedOperatorMap);
+ }
+
@Override
public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc,
AlgebricksPartitionConstraint apcArg) {
@@ -174,6 +184,7 @@
jobSpec.addRoot(opDesc);
}
setAllPartitionConstraints(tgtConstraints);
+ jobSpec.setLogical2PhysicalMap(getLogical2PhysicalMap());
}
public List<IOperatorDescriptor> getGeneratedMetaOps() {
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index fb77ba0..3e51a80 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -47,10 +47,12 @@
private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
private static final String ARRAY_INDEX = "ARRAY_INDEX";
private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
+ private static final String BATCH_LOOKUP = "BATCH_LOOKUP";
private static final String CBO = "CBO";
private static final String CBO_TEST = "CBO_TEST";
private static final String FORCE_JOIN_ORDER = "FORCE_JOIN_ORDER";
private static final String QUERY_PLAN_SHAPE = "QUERY_PLAN_SHAPE";
+ private static final String COLUMN_FILTER = "COLUMN_FILTER";
private final Properties properties = new Properties();
@@ -281,10 +283,26 @@
setString(QUERY_PLAN_SHAPE, queryPlanShape);
}
+ public boolean isBatchLookupEnabled() {
+ return getBoolean(BATCH_LOOKUP, AlgebricksConfig.BATCH_LOOKUP_DEFAULT);
+ }
+
+ public void setBatchLookup(boolean batchedLookup) {
+ setBoolean(BATCH_LOOKUP, batchedLookup);
+ }
+
public void setExternalScanBufferSize(int bufferSize) {
setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
}
+ public void setColumnFilter(boolean columnFilter) {
+ setBoolean(COLUMN_FILTER, columnFilter);
+ }
+
+ public boolean isColumnFilterEnabled() {
+ return getBoolean(COLUMN_FILTER, AlgebricksConfig.COLUMN_FILTER_DEFAULT);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java
index ee7f7aa..75da5ff 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java
@@ -26,6 +26,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -63,6 +64,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -484,6 +486,12 @@
}
@Override
+ public String visitSwitchOperator(SwitchOperator op, Boolean showDetails) throws AlgebricksException {
+ // TODO (GLENN): Implement this logic
+ throw new NotImplementedException();
+ }
+
+ @Override
public String visitMaterializeOperator(MaterializeOperator op, Boolean showDetails) throws AlgebricksException {
stringBuilder.setLength(0);
stringBuilder.append("materialize");
diff --git a/hyracks-fullstack/algebricks/algebricks-data/pom.xml b/hyracks-fullstack/algebricks/algebricks-data/pom.xml
index f48c59ce..e3cfb00 100644
--- a/hyracks-fullstack/algebricks/algebricks-data/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-data/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
index b0ea2fc..95b301f 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
index e13ec30..129aba3 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
@@ -132,6 +132,7 @@
e.setExecutionMode(inOp.getExecutionMode());
context.computeAndSetTypeEnvironmentForOperator(e);
e.recomputeSchema();
+ e.computeDeliveredPhysicalProperties(context);
inOpRef.setValue(e);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
index a6fe495..a1aa01a 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -73,6 +73,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -112,6 +113,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.physical.StreamSelectPOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.physical.StringStreamingScriptPOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.physical.SubplanPOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.physical.SwitchPOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.physical.TokenizePOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.physical.UnionAllPOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.physical.UnnestPOperator;
@@ -309,6 +311,11 @@
}
@Override
+ public IPhysicalOperator visitSwitchOperator(SwitchOperator op, Boolean topLevelOp) {
+ return new SwitchPOperator();
+ }
+
+ @Override
public IPhysicalOperator visitScriptOperator(ScriptOperator op, Boolean topLevelOp) {
return new StringStreamingScriptPOperator();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
index 3b0e9fb..b3748dd 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
@@ -59,6 +59,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -272,6 +273,11 @@
}
@Override
+ public Void visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return null;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java
index 1388ccb..f4f9eda 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java
@@ -56,6 +56,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SwitchOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -186,6 +187,11 @@
}
@Override
+ public ILogicalOperator visitSwitchOperator(SwitchOperator op, Void arg) throws AlgebricksException {
+ return visit(op);
+ }
+
+ @Override
public ILogicalOperator visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
return visit(op);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/pom.xml b/hyracks-fullstack/algebricks/algebricks-runtime/pom.xml
index fb8afca..05a9a9d 100644
--- a/hyracks-fullstack/algebricks/algebricks-runtime/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java
index 07d4e94..c9754f2 100644
--- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java
@@ -28,7 +28,7 @@
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.EnforceFrameWriter;
-import org.apache.hyracks.api.dataflow.TimedFrameWriter;
+import org.apache.hyracks.api.dataflow.ProfiledFrameWriter;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobFlag;
@@ -67,7 +67,7 @@
new RunningAggregatorOutput(ctx, subplans, keyFieldIdx.length + decorFieldIdx.length, writer);
IFrameWriter fw = outputWriter;
if (profile) {
- fw = TimedFrameWriter.time(outputWriter, ctx, "Aggregate Writer");
+ fw = ProfiledFrameWriter.time(outputWriter, ctx, "Aggregate Writer");
} else if (enforce) {
fw = EnforceFrameWriter.enforce(outputWriter);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
index 793f095..d9ba859 100644
--- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
@@ -55,9 +55,12 @@
@Override
public String toString() {
- String s = "stream-limit " + maxObjectsEvalFactory.toString();
+ String s = "stream-limit ";
+ if (maxObjectsEvalFactory != null) {
+ s += maxObjectsEvalFactory.toString();
+ }
if (offsetEvalFactory != null) {
- return s + ", " + offsetEvalFactory.toString();
+ return s + maxObjectsEvalFactory == null ? "" : ", " + offsetEvalFactory.toString();
} else {
return s;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/SwitchOperatorDescriptor.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/SwitchOperatorDescriptor.java
new file mode 100644
index 0000000..0fbb1d5
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/SwitchOperatorDescriptor.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.algebricks.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.hyracks.algebricks.data.IBinaryIntegerInspector;
+import org.apache.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.evaluators.EvaluatorContext;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.ActivityId;
+import org.apache.hyracks.api.dataflow.IActivityGraphBuilder;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.std.base.AbstractActivityNode;
+import org.apache.hyracks.dataflow.std.base.AbstractReplicateOperatorDescriptor;
+import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputOperatorNodePushable;
+
+/**
+ * The switch operator propagates each tuple of the input to one or more output branches based on a given output mapping.
+ * For each tuple, we peek at the value of a given field f. We look up the value of the field in the supplied
+ * output mapping and propagate the tuple to all corresponding output branches.
+ */
+public class SwitchOperatorDescriptor extends AbstractReplicateOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+ final static int SWITCHER_ACTIVITY_ID = 0;
+
+ private final IScalarEvaluatorFactory branchingExprEvalFactory;
+ private final IBinaryIntegerInspectorFactory intInspectorFactory;
+ private final Map<Integer, int[]> outputMapping;
+
+ /**
+ * @param spec
+ * @param rDesc
+ * @param outputArity equal to the number of non-materialized outputs
+ * @param branchingExprEvalFactory containing the index of the relevant field f
+ * @param intInspectorFactory
+ * @param outputMapping the supplied mapping from field values to arrays of output branch numbers
+ */
+ public SwitchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor rDesc, int outputArity,
+ IScalarEvaluatorFactory branchingExprEvalFactory, IBinaryIntegerInspectorFactory intInspectorFactory,
+ Map<Integer, int[]> outputMapping) {
+ super(spec, rDesc, outputArity);
+ if (outputArity != numberOfNonMaterializedOutputs) {
+ throw new IllegalArgumentException();
+ }
+ this.branchingExprEvalFactory = branchingExprEvalFactory;
+ this.intInspectorFactory = intInspectorFactory;
+ this.outputMapping = outputMapping;
+ }
+
+ @Override
+ public void contributeActivities(IActivityGraphBuilder builder) {
+ SwitcherActivityNode sma = new SwitcherActivityNode(new ActivityId(odId, SWITCHER_ACTIVITY_ID));
+ builder.addActivity(this, sma);
+ builder.addSourceEdge(0, sma, 0);
+ for (int i = 0; i < outputArity; i++) {
+ builder.addTargetEdge(i, sma, i);
+ }
+ }
+
+ private final class SwitcherActivityNode extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ public SwitcherActivityNode(ActivityId id) {
+ super(id);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
+ throws HyracksDataException {
+ final IFrameWriter[] writers = new IFrameWriter[outputArity];
+ final boolean[] isOpen = new boolean[outputArity];
+ final IPointable p = VoidPointable.FACTORY.createPointable();
+ // To deal with each tuple in a frame
+ final FrameTupleAccessor accessor = new FrameTupleAccessor(outRecDescs[0]);
+ final FrameTupleAppender[] appenders = new FrameTupleAppender[outputArity];
+ final FrameTupleReference tRef = new FrameTupleReference();
+ final IBinaryIntegerInspector intInspector = intInspectorFactory.createBinaryIntegerInspector(ctx);
+ final IEvaluatorContext evalCtx = new EvaluatorContext(ctx);
+ final IScalarEvaluator eval = branchingExprEvalFactory.createScalarEvaluator(evalCtx);
+ final MutableBoolean hasFailed = new MutableBoolean(false);
+ for (int i = 0; i < outputArity; i++) {
+ appenders[i] = new FrameTupleAppender(new VSizeFrame(ctx), true);
+ }
+
+ return new AbstractUnaryInputOperatorNodePushable() {
+ @Override
+ public void open() throws HyracksDataException {
+ for (int i = 0; i < outputArity; i++) {
+ isOpen[i] = true;
+ writers[i].open();
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer bufferAccessor) throws HyracksDataException {
+ // Tuple based access
+ accessor.reset(bufferAccessor);
+ int tupleCount = accessor.getTupleCount();
+
+ for (int i = 0; i < tupleCount; i++) {
+ // Get the value of the relevant field in the given tuple.
+ tRef.reset(accessor, i);
+ eval.evaluate(tRef, p);
+ int fieldValue =
+ intInspector.getIntegerValue(p.getByteArray(), p.getStartOffset(), p.getLength());
+
+ // Look up the corresponding output branches based on the given mapping
+ int[] outputBranches = outputMapping.get(fieldValue);
+
+ // Propagate to each output branch
+ for (int j : outputBranches) {
+ FrameUtils.appendToWriter(writers[j], appenders[j], accessor, i);
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ Throwable hde = null;
+ // write if hasn't failed
+ if (!hasFailed.booleanValue()) {
+ for (int i = 0; i < outputArity; i++) {
+ if (isOpen[i]) {
+ try {
+ appenders[i].write(writers[i], true);
+ } catch (Throwable th) {
+ hde = th;
+ break;
+ }
+ }
+ }
+ }
+
+ // fail the writers
+ if (hde != null) {
+ for (int i = 0; i < outputArity; i++) {
+ if (isOpen[i]) {
+ CleanupUtils.fail(writers[i], hde);
+ }
+ }
+ }
+
+ // close
+ for (int i = 0; i < outputArity; i++) {
+ if (isOpen[i]) {
+ hde = CleanupUtils.close(writers[i], hde);
+ }
+ }
+ if (hde != null) {
+ throw HyracksDataException.create(hde);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ hasFailed.setTrue();
+ HyracksDataException hde = null;
+ for (int i = 0; i < outputArity; i++) {
+ if (isOpen[i]) {
+ try {
+ writers[i].fail();
+ } catch (Throwable th) {
+ if (hde == null) {
+ hde = HyracksDataException.create(th);
+ } else {
+ hde.addSuppressed(th);
+ }
+ }
+ }
+ }
+ if (hde != null) {
+ throw hde;
+ }
+ }
+
+ @Override
+ public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
+ writers[index] = writer;
+ }
+
+ @Override
+ public void flush() throws HyracksDataException {
+ for (int i = 0; i < outputArity; i++) {
+ if (isOpen[i]) {
+ appenders[i].flush(writers[i]);
+ }
+ }
+ }
+ };
+ }
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-0.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-0.tbl
new file mode 100644
index 0000000..026bbdb
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-0.tbl
@@ -0,0 +1,6 @@
+1,second branch1
+2,third branch1
+1,second branch2
+2,third branch2
+1,second branch3
+2,third branch3
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-1.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-1.tbl
new file mode 100644
index 0000000..464a6448
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-1.tbl
@@ -0,0 +1,7 @@
+0,first branch1
+2,third branch1
+0,first branch2
+2,third branch2
+0,first branch3
+2,third branch3
+0,first branch4
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-2.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-2.tbl
new file mode 100644
index 0000000..b9eabbc
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2-switch-2.tbl
@@ -0,0 +1,7 @@
+0,first branch1
+1,second branch1
+0,first branch2
+1,second branch2
+0,first branch3
+1,second branch3
+0,first branch4
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2.tbl
new file mode 100644
index 0000000..373cd0f
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part2.tbl
@@ -0,0 +1,10 @@
+0|first branch1
+1|second branch1
+2|third branch1
+0|first branch2
+1|second branch2
+2|third branch2
+0|first branch3
+1|second branch3
+2|third branch3
+0|first branch4
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/pom.xml b/hyracks-fullstack/algebricks/algebricks-tests/pom.xml
index d11df8a..a6b1b54 100644
--- a/hyracks-fullstack/algebricks/algebricks-tests/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-tests/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
index aeb22b6..d23f7f9 100644
--- a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
+++ b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
@@ -23,6 +23,7 @@
import java.io.FileReader;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
@@ -58,6 +59,7 @@
import org.apache.hyracks.algebricks.runtime.operators.std.StreamProjectRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.operators.std.StreamSelectRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.operators.std.StringStreamingRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.operators.std.SwitchOperatorDescriptor;
import org.apache.hyracks.algebricks.runtime.operators.std.UnnestRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
import org.apache.hyracks.algebricks.tests.util.AlgebricksHyracksIntegrationUtil;
@@ -698,6 +700,76 @@
}
@Test
+ public void scanSwitchWrite() throws Exception {
+ final int outputArity = 3;
+
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
+
+ String inputFileName[] = { "data" + File.separator + "simple" + File.separator + "int-string-part2.tbl",
+ "data" + File.separator + "simple" + File.separator + "int-string-part2-switch-0.tbl",
+ "data" + File.separator + "simple" + File.separator + "int-string-part2-switch-1.tbl",
+ "data" + File.separator + "simple" + File.separator + "int-string-part2-switch-2.tbl" };
+ File[] inputFiles = new File[inputFileName.length];
+ for (int i = 0; i < inputFileName.length; i++) {
+ inputFiles[i] = new File(inputFileName[i]);
+ }
+ File[] outputFile = new File[outputArity];
+ FileSplit[] outputFileSplit = new FileSplit[outputArity];
+ for (int i = 0; i < outputArity; i++) {
+ outputFileSplit[i] = createFile(AlgebricksHyracksIntegrationUtil.nc1);
+ outputFile[i] = outputFileSplit[i].getFile(AlgebricksHyracksIntegrationUtil.nc1.getIoManager());
+ }
+
+ FileSplit[] inputSplits =
+ new FileSplit[] { new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFileName[0]) };
+ IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(inputSplits);
+
+ RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() });
+
+ IValueParserFactory[] valueParsers =
+ new IValueParserFactory[] { IntegerParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE };
+
+ FileScanOperatorDescriptor intScanner = new FileScanOperatorDescriptor(spec, intSplitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, intScanner, DEFAULT_NODES);
+
+ HashMap<Integer, int[]> outputMapping = new HashMap<>();
+ outputMapping.put(0, new int[] { 1, 2 });
+ outputMapping.put(1, new int[] { 0, 2 });
+ outputMapping.put(2, new int[] { 0, 1 });
+
+ SwitchOperatorDescriptor switchOp = new SwitchOperatorDescriptor(spec, scannerDesc, outputArity,
+ new TupleFieldEvaluatorFactory(0), BinaryIntegerInspectorImpl.FACTORY, outputMapping);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, switchOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length];
+ for (int i = 0; i < outputArity; i++) {
+ outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] { outputFileSplit[i] });
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i],
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+ }
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), intScanner, 0, switchOp, 0);
+ for (int i = 0; i < outputArity; i++) {
+ spec.connect(new OneToOneConnectorDescriptor(spec), switchOp, i, outputOp[i], 0);
+ }
+
+ for (int i = 0; i < outputArity; i++) {
+ spec.addRoot(outputOp[i]);
+ }
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ for (int i = 0; i < outputArity; i++) {
+ compareFiles("data" + File.separator + "device0" + File.separator + inputFileName[i + 1],
+ outputFile[i].getAbsolutePath());
+ }
+ }
+
+ @Test
public void scanMicroSortWrite() throws Exception {
JobSpecification spec = new JobSpecification(FRAME_SIZE);
diff --git a/hyracks-fullstack/algebricks/pom.xml b/hyracks-fullstack/algebricks/pom.xml
index 498581f..2fee451 100644
--- a/hyracks-fullstack/algebricks/pom.xml
+++ b/hyracks-fullstack/algebricks/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>apache-hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks-fullstack-license/pom.xml b/hyracks-fullstack/hyracks-fullstack-license/pom.xml
index 7dd97f9..14037a5 100644
--- a/hyracks-fullstack/hyracks-fullstack-license/pom.xml
+++ b/hyracks-fullstack/hyracks-fullstack-license/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>apache-hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-api/pom.xml b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
index fe8fb24..366b585 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IIntrospectingOperator.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IIntrospectingOperator.java
index 3c1a24d..64897ef 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IIntrospectingOperator.java
@@ -1,3 +1,4 @@
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -16,20 +17,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.hyracks.api.dataflow;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import org.apache.hyracks.api.job.profiling.IOperatorStats;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+public interface IIntrospectingOperator {
+ void setOperatorStats(IOperatorStats stats);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ProfiledFrameWriter.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ProfiledFrameWriter.java
new file mode 100644
index 0000000..5b9495a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ProfiledFrameWriter.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.dataflow;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.comm.FrameConstants;
+import org.apache.hyracks.api.comm.FrameHelper;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.profiling.IOperatorStats;
+import org.apache.hyracks.api.job.profiling.IStatsCollector;
+import org.apache.hyracks.api.job.profiling.OperatorStats;
+import org.apache.hyracks.api.job.profiling.counters.ICounter;
+import org.apache.hyracks.util.IntSerDeUtils;
+
+public class ProfiledFrameWriter implements IFrameWriter, IPassableTimer {
+
+ // The downstream data consumer of this writer.
+ private final IFrameWriter writer;
+ private long frameStart = 0;
+ final ICounter timeCounter;
+ final ICounter tupleCounter;
+ final IStatsCollector collector;
+ final IOperatorStats stats;
+ final IOperatorStats parentStats;
+ private int minSz = Integer.MAX_VALUE;
+ private int maxSz = -1;
+ private long avgSz;
+ final String name;
+
+ public ProfiledFrameWriter(IFrameWriter writer, IStatsCollector collector, String name, IOperatorStats stats,
+ IOperatorStats parentStats) {
+ this.writer = writer;
+ this.collector = collector;
+ this.name = name;
+ this.stats = stats;
+ this.parentStats = parentStats;
+ this.timeCounter = stats.getTimeCounter();
+ this.tupleCounter = parentStats != null ? parentStats.getTupleCounter() : null;
+ }
+
+ @Override
+ public final void open() throws HyracksDataException {
+ try {
+ startClock();
+ writer.open();
+ } finally {
+ stopClock();
+ }
+ }
+
+ @Override
+ public final void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ try {
+ int tupleCountOffset = FrameHelper.getTupleCountOffset(buffer.limit());
+ int tupleCount = IntSerDeUtils.getInt(buffer.array(), tupleCountOffset);
+ if (tupleCounter != null) {
+ long prevCount = tupleCounter.get();
+ for (int i = 0; i < tupleCount; i++) {
+ int tupleLen = getTupleLength(i, tupleCountOffset, buffer);
+ if (maxSz < tupleLen) {
+ maxSz = tupleLen;
+ }
+ if (minSz > tupleLen) {
+ minSz = tupleLen;
+ }
+ long prev = avgSz * prevCount;
+ avgSz = (prev + tupleLen) / (prevCount + 1);
+ prevCount++;
+ }
+ parentStats.getMaxTupleSz().set(maxSz);
+ parentStats.getMinTupleSz().set(minSz);
+ parentStats.getAverageTupleSz().set(avgSz);
+ tupleCounter.update(tupleCount);
+ }
+ startClock();
+ writer.nextFrame(buffer);
+ } finally {
+ stopClock();
+ }
+ }
+
+ @Override
+ public final void flush() throws HyracksDataException {
+ try {
+ startClock();
+ writer.flush();
+ } finally {
+ stopClock();
+ }
+ }
+
+ @Override
+ public final void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ startClock();
+ writer.close();
+ } finally {
+ stopClock();
+ }
+ }
+
+ private void stopClock() {
+ pause();
+ collector.giveClock(this);
+ }
+
+ private void startClock() {
+ if (frameStart > 0) {
+ return;
+ }
+ frameStart = collector.takeClock(this);
+ }
+
+ @Override
+ public void resume() {
+ if (frameStart > 0) {
+ return;
+ }
+ long nt = System.nanoTime();
+ frameStart = nt;
+ }
+
+ @Override
+ public void pause() {
+ if (frameStart > 1) {
+ long nt = System.nanoTime();
+ long delta = nt - frameStart;
+ timeCounter.update(delta);
+ frameStart = -1;
+ }
+ }
+
+ private int getTupleStartOffset(int tupleIndex, int tupleCountOffset, ByteBuffer buffer) {
+ return tupleIndex == 0 ? FrameConstants.TUPLE_START_OFFSET
+ : IntSerDeUtils.getInt(buffer.array(), tupleCountOffset - FrameConstants.SIZE_LEN * tupleIndex);
+ }
+
+ private int getTupleEndOffset(int tupleIndex, int tupleCountOffset, ByteBuffer buffer) {
+ return IntSerDeUtils.getInt(buffer.array(), tupleCountOffset - FrameConstants.SIZE_LEN * (tupleIndex + 1));
+ }
+
+ public int getTupleLength(int tupleIndex, int tupleCountOffset, ByteBuffer buffer) {
+ return getTupleEndOffset(tupleIndex, tupleCountOffset, buffer)
+ - getTupleStartOffset(tupleIndex, tupleCountOffset, buffer);
+ }
+
+ public static IFrameWriter time(IFrameWriter writer, IHyracksTaskContext ctx, String name)
+ throws HyracksDataException {
+ if (!(writer instanceof ProfiledFrameWriter)) {
+ IStatsCollector statsCollector = ctx.getStatsCollector();
+ IOperatorStats stats = new OperatorStats(name);
+ statsCollector.add(stats);
+ return new ProfiledFrameWriter(writer, ctx.getStatsCollector(), name, stats, null);
+
+ } else
+ return writer;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ProfiledOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ProfiledOperatorNodePushable.java
new file mode 100644
index 0000000..f787a1c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ProfiledOperatorNodePushable.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.dataflow;
+
+import java.util.HashMap;
+
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.profiling.IOperatorStats;
+import org.apache.hyracks.api.job.profiling.IStatsCollector;
+import org.apache.hyracks.api.job.profiling.OperatorStats;
+import org.apache.hyracks.api.rewriter.runtime.SuperActivityOperatorNodePushable;
+
+public class ProfiledOperatorNodePushable extends ProfiledFrameWriter implements IOperatorNodePushable, IPassableTimer {
+
+ IOperatorNodePushable op;
+ ProfiledOperatorNodePushable parentOp;
+ ActivityId acId;
+ HashMap<Integer, IFrameWriter> inputs;
+ long frameStart;
+
+ ProfiledOperatorNodePushable(IOperatorNodePushable op, ActivityId acId, IStatsCollector collector,
+ IOperatorStats stats, ActivityId parent, ProfiledOperatorNodePushable parentOp)
+ throws HyracksDataException {
+ super(null, collector, acId.toString() + " - " + op.getDisplayName(), stats,
+ parentOp != null ? parentOp.getStats() : null);
+ this.parentOp = parentOp;
+ this.op = op;
+ this.acId = acId;
+ inputs = new HashMap<>();
+ }
+
+ @Override
+ public void initialize() throws HyracksDataException {
+ synchronized (collector) {
+ startClock();
+ op.initialize();
+ stopClock();
+ }
+ }
+
+ @Override
+ public void deinitialize() throws HyracksDataException {
+ synchronized (collector) {
+ startClock();
+ op.deinitialize();
+ stopClock();
+ }
+ }
+
+ @Override
+ public int getInputArity() {
+ return op.getInputArity();
+ }
+
+ @Override
+ public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc)
+ throws HyracksDataException {
+ op.setOutputFrameWriter(index, writer, recordDesc);
+ }
+
+ @Override
+ public IFrameWriter getInputFrameWriter(int index) {
+ IFrameWriter ifw = op.getInputFrameWriter(index);
+ if (!(op instanceof ProfiledFrameWriter) && ifw.equals(op)) {
+ return new ProfiledFrameWriter(op.getInputFrameWriter(index), collector,
+ acId.toString() + "-" + op.getDisplayName(), stats, parentStats);
+ }
+ return op.getInputFrameWriter(index);
+ }
+
+ @Override
+ public String getDisplayName() {
+ return op.getDisplayName();
+ }
+
+ private void stopClock() {
+ pause();
+ collector.giveClock(this);
+ }
+
+ private void startClock() {
+ if (frameStart > 0) {
+ return;
+ }
+ frameStart = collector.takeClock(this);
+ }
+
+ @Override
+ public void resume() {
+ if (frameStart > 0) {
+ return;
+ }
+ long nt = System.nanoTime();
+ frameStart = nt;
+ }
+
+ @Override
+ public void pause() {
+ if (frameStart > 0) {
+ long nt = System.nanoTime();
+ long delta = nt - frameStart;
+ timeCounter.update(delta);
+ frameStart = -1;
+ }
+ }
+
+ public IOperatorStats getStats() {
+ return stats;
+ }
+
+ public IOperatorStats getParentStats() {
+ return parentStats;
+ }
+
+ public static IOperatorNodePushable time(IOperatorNodePushable op, IHyracksTaskContext ctx, ActivityId acId,
+ ProfiledOperatorNodePushable source) throws HyracksDataException {
+ String name = acId.toString() + " - " + op.getDisplayName();
+ IStatsCollector statsCollector = ctx.getStatsCollector();
+ IOperatorStats stats = new OperatorStats(name, acId.getOperatorDescriptorId());
+ statsCollector.add(stats);
+ if (op instanceof IIntrospectingOperator) {
+ ((IIntrospectingOperator) op).setOperatorStats(stats);
+ }
+ if (!(op instanceof ProfiledOperatorNodePushable) && !(op instanceof SuperActivityOperatorNodePushable)) {
+ return new ProfiledOperatorNodePushable(op, acId, ctx.getStatsCollector(), stats, acId, source);
+ }
+ return op;
+ }
+
+ public static void onlyAddStats(IOperatorNodePushable op, IHyracksTaskContext ctx, ActivityId acId)
+ throws HyracksDataException {
+ String name = acId.toString() + " - " + op.getDisplayName();
+ IStatsCollector statsCollector = ctx.getStatsCollector();
+ IOperatorStats stats = new OperatorStats(name, acId.getOperatorDescriptorId());
+ if (op instanceof IIntrospectingOperator) {
+ ((IIntrospectingOperator) op).setOperatorStats(stats);
+ statsCollector.add(stats);
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/TimedFrameWriter.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/TimedFrameWriter.java
deleted file mode 100644
index 83a4b34..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/TimedFrameWriter.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataflow;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.profiling.IStatsCollector;
-import org.apache.hyracks.api.job.profiling.counters.ICounter;
-
-public class TimedFrameWriter implements IFrameWriter, IPassableTimer {
-
- // The downstream data consumer of this writer.
- private final IFrameWriter writer;
- private long frameStart = 0;
- final ICounter counter;
- final IStatsCollector collector;
- final String name;
-
- public TimedFrameWriter(IFrameWriter writer, IStatsCollector collector, String name, ICounter counter) {
- this.writer = writer;
- this.collector = collector;
- this.name = name;
- this.counter = counter;
- }
-
- protected TimedFrameWriter(IFrameWriter writer, IStatsCollector collector, String name) {
- this(writer, collector, name, collector.getOrAddOperatorStats(name).getTimeCounter());
- }
-
- @Override
- public final void open() throws HyracksDataException {
- try {
- startClock();
- writer.open();
- } finally {
- stopClock();
- }
- }
-
- @Override
- public final void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- try {
- startClock();
- writer.nextFrame(buffer);
- } finally {
- stopClock();
- }
- }
-
- @Override
- public final void flush() throws HyracksDataException {
- try {
- startClock();
- writer.flush();
- } finally {
- stopClock();
- }
- }
-
- @Override
- public final void fail() throws HyracksDataException {
- writer.fail();
- }
-
- @Override
- public void close() throws HyracksDataException {
- try {
- startClock();
- writer.close();
- } finally {
- stopClock();
- }
- }
-
- private void stopClock() {
- pause();
- collector.giveClock(this);
- }
-
- private void startClock() {
- if (frameStart > 0) {
- return;
- }
- frameStart = collector.takeClock(this);
- }
-
- @Override
- public void resume() {
- if (frameStart > 0) {
- return;
- }
- long nt = System.nanoTime();
- frameStart = nt;
- }
-
- @Override
- public void pause() {
- if (frameStart > 1) {
- long nt = System.nanoTime();
- long delta = nt - frameStart;
- counter.update(delta);
- frameStart = -1;
- }
- }
-
- public static IFrameWriter time(IFrameWriter writer, IHyracksTaskContext ctx, String name)
- throws HyracksDataException {
- return writer instanceof TimedFrameWriter ? writer
- : new TimedFrameWriter(writer, ctx.getStatsCollector(), name);
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/TimedOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/TimedOperatorNodePushable.java
deleted file mode 100644
index 2d46bea..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/TimedOperatorNodePushable.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataflow;
-
-import java.util.HashMap;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.profiling.IStatsCollector;
-import org.apache.hyracks.api.rewriter.runtime.SuperActivityOperatorNodePushable;
-
-public class TimedOperatorNodePushable extends TimedFrameWriter implements IOperatorNodePushable, IPassableTimer {
-
- IOperatorNodePushable op;
- HashMap<Integer, IFrameWriter> inputs;
- long frameStart;
-
- TimedOperatorNodePushable(IOperatorNodePushable op, IStatsCollector collector) throws HyracksDataException {
- super(null, collector, op.getDisplayName());
- this.op = op;
- inputs = new HashMap<>();
- }
-
- @Override
- public void initialize() throws HyracksDataException {
- synchronized (collector) {
- startClock();
- op.initialize();
- stopClock();
- }
- }
-
- @Override
- public void deinitialize() throws HyracksDataException {
- synchronized (collector) {
- startClock();
- op.deinitialize();
- stopClock();
- }
- }
-
- @Override
- public int getInputArity() {
- return op.getInputArity();
- }
-
- @Override
- public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc)
- throws HyracksDataException {
- op.setOutputFrameWriter(index, writer, recordDesc);
- }
-
- @Override
- public IFrameWriter getInputFrameWriter(int index) {
- IFrameWriter ifw = op.getInputFrameWriter(index);
- if (!(op instanceof TimedFrameWriter) && ifw.equals(op)) {
- return new TimedFrameWriter(op.getInputFrameWriter(index), collector, op.getDisplayName(), counter);
- }
- return op.getInputFrameWriter(index);
- }
-
- @Override
- public String getDisplayName() {
- return op.getDisplayName();
- }
-
- private void stopClock() {
- pause();
- collector.giveClock(this);
- }
-
- private void startClock() {
- if (frameStart > 0) {
- return;
- }
- frameStart = collector.takeClock(this);
- }
-
- @Override
- public void resume() {
- if (frameStart > 0) {
- return;
- }
- long nt = System.nanoTime();
- frameStart = nt;
- }
-
- @Override
- public void pause() {
- if (frameStart > 0) {
- long nt = System.nanoTime();
- long delta = nt - frameStart;
- counter.update(delta);
- frameStart = -1;
- }
- }
-
- public static IOperatorNodePushable time(IOperatorNodePushable op, IHyracksTaskContext ctx)
- throws HyracksDataException {
- if (!(op instanceof TimedOperatorNodePushable) && !(op instanceof SuperActivityOperatorNodePushable)) {
- return new TimedOperatorNodePushable(op, ctx.getStatsCollector());
- }
- return op;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
index 58336a0..2c51d3d 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -49,7 +50,7 @@
import com.fasterxml.jackson.databind.node.ObjectNode;
public class JobSpecification implements Serializable, IOperatorDescriptorRegistry, IConnectorDescriptorRegistry {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private static final int DEFAULT_FRAME_SIZE = 32768;
@@ -67,6 +68,8 @@
private final Map<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>> connectorOpMap;
+ private transient Map<Object, String> logical2PhysicalMap;
+
private final Map<String, Serializable> properties;
private final Set<Constraint> userConstraints;
@@ -108,6 +111,7 @@
opOutputMap = new HashMap<>();
connectorOpMap = new HashMap<>();
properties = new HashMap<>();
+ logical2PhysicalMap = Collections.emptyMap();
userConstraints = new HashSet<>();
operatorIdCounter = 0;
connectorIdCounter = 0;
@@ -326,6 +330,14 @@
return metaOps;
}
+ public void setLogical2PhysicalMap(Map<Object, String> logical2PhysicalMap) {
+ this.logical2PhysicalMap = logical2PhysicalMap;
+ }
+
+ public Map<Object, String> getLogical2PhysicalMap() {
+ return logical2PhysicalMap;
+ }
+
private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
List<V> vList = map.computeIfAbsent(key, k -> new ArrayList<>());
extend(vList, index);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
index 0d38fac..26b3a58 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
@@ -21,6 +21,7 @@
import java.io.Serializable;
import java.util.Map;
+import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.io.IWritable;
import org.apache.hyracks.api.job.profiling.counters.ICounter;
@@ -44,9 +45,48 @@
ICounter getTimeCounter();
/**
- * @return A counter used to track the number of pages pinned by an opeartor
+ * @return A counter used to track the number of pages pinned by an operator
*/
- ICounter getDiskIoCounter();
+ ICounter getPageReads();
+
+ /**
+ * @return A counter used to track the number of pages read from disk by an operator
+ */
+
+ ICounter coldReadCounter();
+
+ /**
+ * @return A counter used to set the average tuple size outputted by an operator
+ */
+
+ ICounter getAverageTupleSz();
+
+ /**
+ * @return A counter used to set the max tuple size outputted by an operator
+ */
+
+ ICounter getMaxTupleSz();
+
+ /**
+ * @return A counter used to set the min tuple size outputted by an operator
+ */
+
+ ICounter getMinTupleSz();
+
+ /**
+ * @return A counter used to track the number of tuples read by operators that originate data,
+ * like index searches or other scan types
+ */
+
+ ICounter getInputTupleCounter();
+
+ ICounter getLevel();
+
+ ICounter getBytesRead();
+
+ ICounter getBytesWritten();
+
+ OperatorDescriptorId getId();
void updateIndexesStats(Map<String, IndexStats> indexesStats);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IStatsCollector.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IStatsCollector.java
index c903c68..d2ad20c 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IStatsCollector.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IStatsCollector.java
@@ -38,14 +38,7 @@
/**
* @param operatorName
* @return {@link IOperatorStats} for the operator with name <code>operatorName</code>
- * if it already exists, and adds it if it does not.
- */
- IOperatorStats getOrAddOperatorStats(String operatorName);
-
- /**
- * @param operatorName
- * @return {@link IOperatorStats} for the operator with name <code>operatorName</code>
- * if it already exists, or {@code null} if the operator does not exist
+ * if one exists or else null.
*/
IOperatorStats getOperatorStats(String operatorName);
@@ -74,5 +67,4 @@
* @param currHolder the timer that needs to be paused
*/
void giveClock(IPassableTimer currHolder);
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
new file mode 100644
index 0000000..4c86a15
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.job.profiling;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
+import org.apache.hyracks.api.job.profiling.counters.ICounter;
+
+public class NoOpOperatorStats implements IOperatorStats {
+
+ private static final long serialVersionUID = 9055940822300360135L;
+
+ public static final NoOpOperatorStats INSTANCE = new NoOpOperatorStats();
+
+ private static final ICounter NOOP_COUNTER = new ICounter() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long update(long delta) {
+ return 0;
+ }
+
+ @Override
+ public long set(long value) {
+ return 0;
+ }
+
+ @Override
+ public String getName() {
+ return null;
+ }
+
+ @Override
+ public long get() {
+ return 0;
+ }
+ };
+
+ private static final OperatorDescriptorId INVALID_ODID = new OperatorDescriptorId(-1);
+
+ @Override
+ public void writeFields(DataOutput output) throws IOException {
+
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+
+ }
+
+ @Override
+ public String getName() {
+ return "";
+ }
+
+ @Override
+ public ICounter getTupleCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getTimeCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getPageReads() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter coldReadCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getAverageTupleSz() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getMaxTupleSz() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getMinTupleSz() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getInputTupleCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getLevel() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getBytesRead() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getBytesWritten() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public OperatorDescriptorId getId() {
+ return INVALID_ODID;
+ }
+
+ @Override
+ public void updateIndexesStats(Map<String, IndexStats> indexesStats) {
+ // no op
+ }
+
+ @Override
+ public Map<String, IndexStats> getIndexesStats() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public void updateFrom(IOperatorStats stats) {
+ // no op
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
index a3a5073..c9f08e0 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
@@ -25,28 +25,54 @@
import java.util.Map;
import org.apache.hyracks.api.com.job.profiling.counters.Counter;
+import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.job.profiling.counters.ICounter;
public class OperatorStats implements IOperatorStats {
- private static final long serialVersionUID = 6401830963367567167L;
+ private static final long serialVersionUID = 6401830963367567169L;
public final String operatorName;
+ public OperatorDescriptorId id;
public final ICounter tupleCounter;
public final ICounter timeCounter;
- public final ICounter diskIoCounter;
+ public final ICounter pageReads;
+ public final ICounter coldReadCounter;
+ public final ICounter avgTupleSz;
+ public final ICounter minTupleSz;
+ public final ICounter maxTupleSz;
+ public final ICounter inputTupleCounter;
+ public final ICounter level;
+ public final ICounter bytesRead;
+ public final ICounter bytesWritten;
private final Map<String, IndexStats> indexesStats;
- public OperatorStats(String operatorName) {
+ //TODO: this is quickly becoming gross it should just be a map where the value is obliged to be a Counter
+
+ public OperatorStats(String operatorName, OperatorDescriptorId id) {
if (operatorName == null || operatorName.isEmpty()) {
throw new IllegalArgumentException("operatorName must not be null or empty");
}
this.operatorName = operatorName;
+ this.id = id;
tupleCounter = new Counter("tupleCounter");
timeCounter = new Counter("timeCounter");
- diskIoCounter = new Counter("diskIoCounter");
+ pageReads = new Counter("diskIoCounter");
+ coldReadCounter = new Counter("coldReadCounter");
+ avgTupleSz = new Counter("avgTupleSz");
+ minTupleSz = new Counter("minTupleSz");
+ maxTupleSz = new Counter("maxTupleSz");
+ inputTupleCounter = new Counter("inputTupleCounter");
+ level = new Counter("level");
+ bytesRead = new Counter("bytesRead");
+ bytesWritten = new Counter("bytesWritten");
+ level.set(-1);
indexesStats = new HashMap<>();
}
+ public OperatorStats(String operatorName) {
+ this(operatorName, new OperatorDescriptorId(-1));
+ }
+
public static IOperatorStats create(DataInput input) throws IOException {
String name = input.readUTF();
OperatorStats operatorStats = new OperatorStats(name);
@@ -70,8 +96,53 @@
}
@Override
- public ICounter getDiskIoCounter() {
- return diskIoCounter;
+ public ICounter getPageReads() {
+ return pageReads;
+ }
+
+ @Override
+ public ICounter coldReadCounter() {
+ return coldReadCounter;
+ }
+
+ @Override
+ public ICounter getAverageTupleSz() {
+ return avgTupleSz;
+ }
+
+ @Override
+ public ICounter getMaxTupleSz() {
+ return maxTupleSz;
+ }
+
+ @Override
+ public ICounter getMinTupleSz() {
+ return minTupleSz;
+ }
+
+ @Override
+ public ICounter getInputTupleCounter() {
+ return inputTupleCounter;
+ }
+
+ @Override
+ public ICounter getLevel() {
+ return level;
+ }
+
+ @Override
+ public ICounter getBytesRead() {
+ return bytesRead;
+ }
+
+ @Override
+ public ICounter getBytesWritten() {
+ return bytesWritten;
+ }
+
+ @Override
+ public OperatorDescriptorId getId() {
+ return id;
}
@Override
@@ -100,24 +171,42 @@
public void updateFrom(IOperatorStats stats) {
tupleCounter.update(stats.getTupleCounter().get());
timeCounter.update(stats.getTimeCounter().get());
- diskIoCounter.update(stats.getDiskIoCounter().get());
+ pageReads.update(stats.getPageReads().get());
updateIndexesStats(stats.getIndexesStats());
}
@Override
public void writeFields(DataOutput output) throws IOException {
output.writeUTF(operatorName);
+ id.writeFields(output);
output.writeLong(tupleCounter.get());
output.writeLong(timeCounter.get());
- output.writeLong(diskIoCounter.get());
+ output.writeLong(pageReads.get());
+ output.writeLong(coldReadCounter.get());
+ output.writeLong(avgTupleSz.get());
+ output.writeLong(minTupleSz.get());
+ output.writeLong(maxTupleSz.get());
+ output.writeLong(inputTupleCounter.get());
+ output.writeLong(level.get());
+ output.writeLong(bytesRead.get());
+ output.writeLong(bytesWritten.get());
writeIndexesStats(output);
}
@Override
public void readFields(DataInput input) throws IOException {
+ id = OperatorDescriptorId.create(input);
tupleCounter.set(input.readLong());
timeCounter.set(input.readLong());
- diskIoCounter.set(input.readLong());
+ pageReads.set(input.readLong());
+ coldReadCounter.set(input.readLong());
+ avgTupleSz.set(input.readLong());
+ minTupleSz.set(input.readLong());
+ maxTupleSz.set(input.readLong());
+ inputTupleCounter.set(input.readLong());
+ level.set(input.readLong());
+ bytesRead.set(input.readLong());
+ bytesWritten.set(input.readLong());
readIndexesStats(input);
}
@@ -142,6 +231,11 @@
public String toString() {
return "{ " + "\"operatorName\": \"" + operatorName + "\", " + "\"" + tupleCounter.getName() + "\": "
+ tupleCounter.get() + ", \"" + timeCounter.getName() + "\": " + timeCounter.get() + ", \""
- + ", \"indexStats\": \"" + indexesStats + "\" }";
+ + coldReadCounter.getName() + "\": " + coldReadCounter.get() + avgTupleSz.getName() + "\": "
+ + avgTupleSz.get() + ", \"" + minTupleSz.getName() + "\": " + minTupleSz.get() + ", \""
+ + minTupleSz.getName() + "\": " + timeCounter.get() + ", \"" + inputTupleCounter.getName() + "\": "
+ + bytesRead.get() + ", \"" + bytesRead.getName() + "\": " + bytesWritten.get() + ", \""
+ + bytesWritten.getName() + "\": " + inputTupleCounter.get() + ", \"" + level.getName() + "\": "
+ + level.get() + ", \"indexStats\": \"" + indexesStats + "\" }";
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
index e43d72a..efd4e07 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
@@ -43,7 +43,7 @@
import org.apache.hyracks.api.dataflow.IActivity;
import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
-import org.apache.hyracks.api.dataflow.TimedOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.ProfiledOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -109,10 +109,12 @@
for (Entry<ActivityId, IActivity> entry : startActivities.entrySet()) {
IOperatorNodePushable opPushable = null;
if (profile) {
- opPushable = TimedOperatorNodePushable
- .time(entry.getValue().createPushRuntime(ctx, recordDescProvider, partition, nPartitions), ctx);
+ IOperatorNodePushable wrapped =
+ entry.getValue().createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
+ opPushable = ProfiledOperatorNodePushable.time(wrapped, ctx, entry.getKey(), null);
} else {
opPushable = entry.getValue().createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
+ ProfiledOperatorNodePushable.onlyAddStats(opPushable, ctx, entry.getKey());
}
operatorNodePushablesBFSOrder.add(opPushable);
operatorNodePushables.put(entry.getKey(), opPushable);
@@ -141,11 +143,18 @@
IOperatorNodePushable destOp = operatorNodePushables.get(destId);
if (destOp == null) {
if (profile) {
- destOp = TimedOperatorNodePushable.time(channel.getRight().getLeft().createPushRuntime(ctx,
- recordDescProvider, partition, nPartitions), ctx);
+ IOperatorNodePushable wrapped = channel.getRight().getLeft().createPushRuntime(ctx,
+ recordDescProvider, partition, nPartitions);
+ if (sourceOp instanceof ProfiledOperatorNodePushable) {
+ destOp = ProfiledOperatorNodePushable.time(wrapped, ctx, destId,
+ (ProfiledOperatorNodePushable) sourceOp);
+ } else {
+ destOp = ProfiledOperatorNodePushable.time(wrapped, ctx, destId, null);
+ }
} else {
destOp = channel.getRight().getLeft().createPushRuntime(ctx, recordDescProvider, partition,
nPartitions);
+ ProfiledOperatorNodePushable.onlyAddStats(destOp, ctx, destId);
}
operatorNodePushablesBFSOrder.add(destOp);
operatorNodePushables.put(destId, destOp);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
index c34a671..3c829e6 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
@@ -28,6 +28,8 @@
public static final String INDEX_CURSOR_STATS = "INDEX_CURSOR_STATS";
+ public static final String TUPLE_PROJECTOR = "TUPLE_PROJECTOR";
+
private HyracksConstants() {
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-client/pom.xml b/hyracks-fullstack/hyracks/hyracks-client/pom.xml
index 66b71fa..c5fed16 100644
--- a/hyracks-fullstack/hyracks/hyracks-client/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-client/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-comm/pom.xml b/hyracks-fullstack/hyracks/hyracks-comm/pom.xml
index f73060f..4f34490 100644
--- a/hyracks-fullstack/hyracks/hyracks-comm/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-comm/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
index 57e22e7..01f426c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-control</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobRun.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobRun.java
index 0cc09b4..d865b4f 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobRun.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobRun.java
@@ -111,10 +111,11 @@
pmm = new PartitionMatchMaker();
participatingNodeIds = new HashSet<>();
cleanupPendingNodeIds = new HashSet<>();
- profile = new JobProfile(jobId);
connectorPolicyMap = new HashMap<>();
operatorLocations = new HashMap<>();
createTime = System.currentTimeMillis();
+ profile = new JobProfile(jobId);
+ profile.setCreateTime(createTime);
}
//Run a deployed job spec
@@ -200,6 +201,7 @@
public void setStartTime(long startTime) {
this.startTime = startTime;
+ this.profile.setStartTime(startTime);
}
public String getStartTimeZoneId() {
@@ -208,6 +210,7 @@
public void setStartTimeZoneId(String startTimeZoneId) {
this.startTimeZoneId = startTimeZoneId;
+ this.profile.setStartTimeZoneId(startTimeZoneId);
}
public long getEndTime() {
@@ -216,6 +219,7 @@
public void setEndTime(long endTime) {
this.endTime = endTime;
+ this.profile.setEndTime(endTime);
}
public void registerOperatorLocation(OperatorDescriptorId op, int partition, String location) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml
index 65152a9..167ebc2 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-control</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 01cb9bf..bb40e2b 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -99,6 +99,7 @@
PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true),
PYTHON_ARGS(STRING_ARRAY, (String[]) null),
PYTHON_ENV(STRING_ARRAY, (String[]) null),
+ PYTHON_DS_PATH(STRING, (String) null),
CREDENTIAL_FILE(
OptionTypes.STRING,
(Function<IApplicationConfig, String>) appConfig -> FileUtil
@@ -248,6 +249,8 @@
return "Whether or not to attempt to automatically set PYTHON_CMD to a usable interpreter";
case PYTHON_ENV:
return "List of environment variables to set when invoking the Python interpreter for Python UDFs. E.g. FOO=1";
+ case PYTHON_DS_PATH:
+ return "Path to systemd socket for fenced Python UDFs. Requires JDK17+, *nix operating system, and ";
case CREDENTIAL_FILE:
return "Path to HTTP basic credentials";
default:
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
index 580a560..41beac0 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
@@ -30,10 +30,11 @@
import org.apache.hyracks.api.dataflow.IPassableTimer;
import org.apache.hyracks.api.job.profiling.IOperatorStats;
import org.apache.hyracks.api.job.profiling.IStatsCollector;
+import org.apache.hyracks.api.job.profiling.NoOpOperatorStats;
import org.apache.hyracks.api.job.profiling.OperatorStats;
public class StatsCollector implements IStatsCollector {
- private static final long serialVersionUID = 6858817639895434578L;
+ private static final long serialVersionUID = 6858817639895434572L;
private final Map<String, IOperatorStats> operatorStatsMap = new LinkedHashMap<>();
private transient Deque<IPassableTimer> clockHolder = new ArrayDeque<>();
@@ -47,13 +48,8 @@
}
@Override
- public IOperatorStats getOrAddOperatorStats(String operatorName) {
- return operatorStatsMap.computeIfAbsent(operatorName, OperatorStats::new);
- }
-
- @Override
public IOperatorStats getOperatorStats(String operatorName) {
- return operatorStatsMap.get(operatorName);
+ return operatorStatsMap.getOrDefault(operatorName, NoOpOperatorStats.INSTANCE);
}
@Override
@@ -71,9 +67,9 @@
public IOperatorStats getAggregatedStats() {
IOperatorStats aggregatedStats = new OperatorStats("aggregated");
for (IOperatorStats stats : operatorStatsMap.values()) {
- aggregatedStats.getTupleCounter().update(stats.getTupleCounter().get());
+ aggregatedStats.getInputTupleCounter().update(stats.getInputTupleCounter().get());
aggregatedStats.getTimeCounter().update(stats.getTimeCounter().get());
- aggregatedStats.getDiskIoCounter().update(stats.getDiskIoCounter().get());
+ aggregatedStats.getPageReads().update(stats.getPageReads().get());
}
return aggregatedStats;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/JobProfile.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/JobProfile.java
index ee49908..f4cd4d6 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/JobProfile.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/JobProfile.java
@@ -32,6 +32,7 @@
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.profiling.IOperatorStats;
import org.apache.hyracks.api.job.profiling.IStatsCollector;
+import org.apache.hyracks.api.job.profiling.NoOpOperatorStats;
import org.apache.hyracks.api.job.profiling.OperatorStats;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -42,6 +43,10 @@
private static final long serialVersionUID = 2L;
private JobId jobId;
+ private long createTime;
+ private long startTime;
+ private String startTimeZoneId;
+ private long endTime;
private Map<String, JobletProfile> jobletProfiles;
@@ -68,12 +73,32 @@
return jobletProfiles;
}
+ public void setCreateTime(long createTime) {
+ this.createTime = createTime;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public void setStartTimeZoneId(String startTimeZoneId) {
+ this.startTimeZoneId = startTimeZoneId;
+ }
+
+ public void setEndTime(long endTime) {
+ this.endTime = endTime;
+ }
+
@Override
public ObjectNode toJSON() {
ObjectMapper om = new ObjectMapper();
ObjectNode json = om.createObjectNode();
json.put("job-id", jobId.toString());
+ json.put("create-time", createTime);
+ json.put("start-time", startTime);
+ json.put("queued-time", startTime - createTime);
+ json.put("end-time", endTime);
populateCounters(json);
ArrayNode jobletsArray = om.createArrayNode();
for (JobletProfile p : jobletProfiles.values()) {
@@ -98,6 +123,10 @@
@Override
public void readFields(DataInput input) throws IOException {
jobId = JobId.create(input);
+ createTime = input.readLong();
+ startTime = input.readLong();
+ endTime = input.readLong();
+ startTimeZoneId = input.readUTF();
int size = input.readInt();
jobletProfiles = new HashMap<>();
for (int i = 0; i < size; i++) {
@@ -110,6 +139,10 @@
@Override
public void writeFields(DataOutput output) throws IOException {
jobId.writeFields(output);
+ output.writeLong(createTime);
+ output.writeLong(startTime);
+ output.writeLong(endTime);
+ output.writeUTF(startTimeZoneId);
output.writeInt(jobletProfiles.size());
for (Entry<String, JobletProfile> entry : jobletProfiles.entrySet()) {
output.writeUTF(entry.getKey());
@@ -141,7 +174,7 @@
for (int i = 0; i < n; i++) {
String operatorName = operatorNames.get(i);
IOperatorStats opTaskStats = statsCollector.getOperatorStats(operatorName);
- if (opTaskStats == null) {
+ if (opTaskStats.equals(NoOpOperatorStats.INSTANCE)) {
continue;
}
IOperatorStats opOutStats = outStats[i];
@@ -154,4 +187,5 @@
}
return Arrays.asList(outStats);
}
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/TaskProfile.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/TaskProfile.java
index dab6d26..4036f00 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/TaskProfile.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/om/TaskProfile.java
@@ -130,7 +130,28 @@
jpe.put("name", key);
jpe.put("time", Double
.parseDouble(new DecimalFormat("#.####").format((double) value.getTimeCounter().get() / 1000000)));
- jpe.put("disk-io", value.getDiskIoCounter().get());
+ if (value.getId().getId() >= 0) {
+ jpe.put("runtime-id", value.getId().toString());
+ }
+ if (value.getPageReads().get() > 0) {
+ jpe.put("pages-read", value.getPageReads().get());
+ jpe.put("pages-read-cold", value.coldReadCounter().get());
+ }
+ if (value.getTupleCounter().get() > 0) {
+ jpe.put("cardinality-out", value.getTupleCounter().get());
+ jpe.put("avg-tuple-size", value.getAverageTupleSz().get());
+ jpe.put("min-tuple-size", value.getMinTupleSz().get());
+ jpe.put("max-tuple-size", value.getMaxTupleSz().get());
+ }
+ if (value.getLevel().get() > -1) {
+ jpe.put("level", value.getLevel().get());
+ }
+ if (value.getBytesRead().get() > 0) {
+ jpe.put("bytes-read", value.getBytesRead().get());
+ }
+ if (value.getBytesWritten().get() > 0) {
+ jpe.put("bytes-written", value.getBytesWritten().get());
+ }
countersObj.add(jpe);
});
json.set("counters", countersObj);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
index 083d268..d68c291 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-control</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml
index db09f40..9852592 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hyracks-control</artifactId>
<groupId>org.apache.hyracks</groupId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/pom.xml
index bf8bb52..989a514 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
index dfc894f..4b88cc8 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-data</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
<root.dir>${basedir}/../../..</root.dir>
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
index 49f6221..8013e05 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
@@ -33,6 +33,7 @@
import org.apache.hyracks.data.std.api.IHashable;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.api.IPointableFactory;
+import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.util.GrowableArray;
import org.apache.hyracks.data.std.util.UTF8StringBuilder;
import org.apache.hyracks.util.string.UTF8StringUtil;
@@ -108,8 +109,7 @@
* Returns the character at the given byte offset. The caller is responsible for making sure that
* the provided offset is within bounds and points to the beginning of a valid UTF8 character.
*
- * @param offset
- * - Byte offset
+ * @param offset - Byte offset
* @return Character at the given offset.
*/
public char charAt(int offset) {
@@ -218,13 +218,15 @@
pointable2.utf8Length);
}
+ public static int compare(IValueReference valueA, IValueReference valueB) {
+ return UTF8StringUtil.compareTo(valueA.getByteArray(), valueA.getStartOffset(), valueA.getLength(),
+ valueB.getByteArray(), valueB.getStartOffset(), valueB.getLength());
+ }
+
/**
- * @param src,
- * the source string.
- * @param pattern,
- * the pattern string.
- * @param ignoreCase,
- * to ignore case or not.
+ * @param src, the source string.
+ * @param pattern, the pattern string.
+ * @param ignoreCase, to ignore case or not.
* @return the byte offset of the first character of the matching string. Not including the MetaLength.
*/
public static int find(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase) {
@@ -232,12 +234,9 @@
}
/**
- * @param src,
- * the source string.
- * @param pattern,
- * the pattern string.
- * @param ignoreCase,
- * to ignore case or not.
+ * @param src, the source string.
+ * @param pattern, the pattern string.
+ * @param ignoreCase, to ignore case or not.
* @return the offset in the unit of code point of the first character of the matching string. Not including the MetaLength.
*/
public static int findInCodePoint(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase) {
@@ -245,30 +244,22 @@
}
/**
- * @param src,
- * the source string.
- * @param pattern,
- * the pattern string.
- * @param ignoreCase,
- * to ignore case or not.
- * @param startMatch,
- * the start offset.
+ * @param src, the source string.
+ * @param pattern, the pattern string.
+ * @param ignoreCase, to ignore case or not.
+ * @param startMatch, the start offset.
* @return the byte offset of the first character of the matching string after <code>startMatchPos}</code>.
- * Not including the MetaLength.
+ * Not including the MetaLength.
*/
public static int find(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase, int startMatch) {
return findInByteOrCodePoint(src, pattern, ignoreCase, startMatch, true);
}
/**
- * @param src,
- * the source string.
- * @param pattern,
- * the pattern string.
- * @param ignoreCase,
- * to ignore case or not.
- * @param startMatch,
- * the start offset.
+ * @param src, the source string.
+ * @param pattern, the pattern string.
+ * @param ignoreCase, to ignore case or not.
+ * @param startMatch, the start offset.
* @return the offset in the unit of code point of the first character of the matching string. Not including the MetaLength.
*/
public static int findInCodePoint(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase,
@@ -324,7 +315,7 @@
}
// The result is counted in code point instead of bytes
- if (resultInByte == false) {
+ if (!resultInByte) {
char ch = src.charAt(srcStart + startMatchPos);
if (Character.isHighSurrogate(ch)) {
prevHighSurrogate = true;
@@ -431,9 +422,10 @@
/**
* Return the substring. Note that the offset and length are in the unit of code point.
+ *
* @return {@code true} if substring was successfully written into given {@code out}, or
- * {@code false} if substring could not be obtained ({@code codePointOffset} or {@code codePointLength}
- * are less than 0 or starting position is greater than the input length)
+ * {@code false} if substring could not be obtained ({@code codePointOffset} or {@code codePointLength}
+ * are less than 0 or starting position is greater than the input length)
*/
public boolean substr(int codePointOffset, int codePointLength, UTF8StringBuilder builder, GrowableArray out)
throws IOException {
@@ -442,9 +434,10 @@
/**
* Return the substring. Note that the offset and length are in the unit of code point.
+ *
* @return {@code true} if substring was successfully written into given {@code out}, or
- * {@code false} if substring could not be obtained ({@code codePointOffset} or {@code codePointLength}
- * are less than 0 or starting position is greater than the input length)
+ * {@code false} if substring could not be obtained ({@code codePointOffset} or {@code codePointLength}
+ * are less than 0 or starting position is greater than the input length)
*/
public static boolean substr(UTF8StringPointable src, int codePointOffset, int codePointLength,
UTF8StringBuilder builder, GrowableArray out) throws IOException {
@@ -548,12 +541,9 @@
/**
* Generates a lower case string of an input string.
*
- * @param src
- * , the input source string.
- * @param builder
- * , a builder for the resulting string.
- * @param out
- * , the storage for a result string.
+ * @param src , the input source string.
+ * @param builder , a builder for the resulting string.
+ * @param out , the storage for a result string.
* @throws IOException
*/
public static void lowercase(UTF8StringPointable src, UTF8StringBuilder builder, GrowableArray out)
@@ -577,12 +567,9 @@
/**
* Generates an upper case string of an input string.
*
- * @param src
- * , the input source string.
- * @param builder
- * , a builder for the resulting string.
- * @param out
- * , the storage for a result string.
+ * @param src , the input source string.
+ * @param builder , a builder for the resulting string.
+ * @param out , the storage for a result string.
* @throws IOException
*/
public static void uppercase(UTF8StringPointable src, UTF8StringBuilder builder, GrowableArray out)
@@ -607,12 +594,9 @@
* Generates a "title" format string from an input source string, i.e., the first letter of each word
* is in the upper case while the other letter is in the lower case.
*
- * @param src
- * , the input source string.
- * @param builder
- * , a builder for the resulting string.
- * @param out
- * , the storage for a result string.
+ * @param src , the input source string.
+ * @param builder , a builder for the resulting string.
+ * @param out , the storage for a result string.
* @throws IOException
*/
public static void initCap(UTF8StringPointable src, UTF8StringBuilder builder, GrowableArray out)
@@ -642,18 +626,12 @@
/**
* Generates a trimmed string of an input source string.
*
- * @param srcPtr
- * , the input source string
- * @param builder
- * , the result string builder.
- * @param out
- * , the storage for the output string.
- * @param left
- * , whether to trim the left side.
- * @param right
- * , whether to trim the right side.
- * @param codePointSet
- * , the set of code points that should be trimmed.
+ * @param srcPtr , the input source string
+ * @param builder , the result string builder.
+ * @param out , the storage for the output string.
+ * @param left , whether to trim the left side.
+ * @param right , whether to trim the right side.
+ * @param codePointSet , the set of code points that should be trimmed.
* @throws IOException
*/
public static void trim(UTF8StringPointable srcPtr, UTF8StringBuilder builder, GrowableArray out, boolean left,
@@ -696,16 +674,11 @@
/**
* Generates a trimmed string from the original string.
*
- * @param builder
- * , the result string builder.
- * @param out
- * , the storage for the output string.
- * @param left
- * , whether to trim the left side.
- * @param right
- * , whether to trim the right side.
- * @param codePointSet
- * , the set of code points that should be trimmed.
+ * @param builder , the result string builder.
+ * @param out , the storage for the output string.
+ * @param left , whether to trim the left side.
+ * @param right , whether to trim the right side.
+ * @param codePointSet , the set of code points that should be trimmed.
* @throws IOException
*/
public void trim(UTF8StringBuilder builder, GrowableArray out, boolean left, boolean right,
@@ -716,12 +689,9 @@
/**
* Generates a reversed string from an input source string
*
- * @param srcPtr
- * , the input source string.
- * @param builder
- * , a builder for the resulting string.
- * @param out
- * , the storage for a result string.
+ * @param srcPtr , the input source string.
+ * @param builder , a builder for the resulting string.
+ * @param out , the storage for a result string.
* @throws IOException
*/
public static void reverse(UTF8StringPointable srcPtr, UTF8StringBuilder builder, GrowableArray out)
@@ -739,7 +709,7 @@
cursorIndex--;
if (UTF8StringUtil.isCharStart(srcPtr.bytes, cursorIndex)) {
ch = UTF8StringUtil.charAt(srcPtr.bytes, cursorIndex);
- if (Character.isHighSurrogate(ch) == false) {
+ if (!Character.isHighSurrogate(ch)) {
throw new IllegalArgumentException(
"Decoding Error: no corresponding high surrogate found for the following low surrogate");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java
index d5a4481..d4feff6 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java
@@ -65,8 +65,12 @@
}
public void append(IValueReference value) throws HyracksDataException {
+ append(value.getByteArray(), value.getStartOffset(), value.getLength());
+ }
+
+ public void append(byte[] bytes, int start, int length) throws HyracksDataException {
try {
- data.append(value);
+ data.append(bytes, start, length);
} catch (IOException e) {
throw HyracksDataException.create(e);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-data/pom.xml b/hyracks-fullstack/hyracks/hyracks-data/pom.xml
index 3d6f62f..45084d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-data/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-common/pom.xml
index 274cad7..059d52a 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
index 284aa0d..6789025 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorNodePushable.java
index 3fa88bb..5559663 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorNodePushable.java
@@ -18,11 +18,18 @@
*/
package org.apache.hyracks.dataflow.std.base;
+import org.apache.hyracks.api.dataflow.IIntrospectingOperator;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.job.profiling.IOperatorStats;
-public abstract class AbstractOperatorNodePushable implements IOperatorNodePushable {
+public abstract class AbstractOperatorNodePushable implements IOperatorNodePushable, IIntrospectingOperator {
@Override
public String getDisplayName() {
return toString();
}
+
+ @Override
+ public void setOperatorStats(IOperatorStats stats) {
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java
index 4c728ce..666c250 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java
@@ -35,10 +35,11 @@
import org.apache.hyracks.dataflow.std.misc.MaterializerTaskState;
/**
- * Abstract class for two replication related operator descriptor - replicate and split
+ * Abstract class for three replication related operator descriptors - replicate, split, and switch.
* Replicate operator propagates all frames to all output branches.
* That is, each tuple will be propagated to all output branches.
* Split operator propagates each tuple in a frame to one output branch only.
+ * Switch is a generalization of split that propagates tuples based on a given output mapping.
*/
public abstract class AbstractReplicateOperatorDescriptor extends AbstractOperatorDescriptor {
protected static final long serialVersionUID = 1L;
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/IPartitionedTupleBufferManager.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/IPartitionedTupleBufferManager.java
index 3645855..190baa2 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/IPartitionedTupleBufferManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/IPartitionedTupleBufferManager.java
@@ -117,9 +117,10 @@
*
* @param pid
* @param writer
+ * @return number of bytes spilled
* @throws HyracksDataException
*/
- void flushPartition(int pid, IFrameWriter writer) throws HyracksDataException;
+ int flushPartition(int pid, IFrameWriter writer) throws HyracksDataException;
/**
* Clear the memory occupation of the particular partition.
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/VPartitionTupleBufferManager.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/VPartitionTupleBufferManager.java
index 722512a..9cabb8a 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/VPartitionTupleBufferManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/VPartitionTupleBufferManager.java
@@ -290,17 +290,20 @@
}
@Override
- public void flushPartition(int pid, IFrameWriter writer) throws HyracksDataException {
+ public int flushPartition(int pid, IFrameWriter writer) throws HyracksDataException {
IFrameBufferManager partition = partitionArray[pid];
+ int written = 0;
if (partition != null && getNumTuples(pid) > 0) {
for (int i = 0; i < partition.getNumFrames(); ++i) {
partition.getFrame(i, tempInfo);
tempInfo.getBuffer().position(tempInfo.getStartOffset());
tempInfo.getBuffer().limit(tempInfo.getStartOffset() + tempInfo.getLength());
+ int sz = tempInfo.getLength();
writer.nextFrame(tempInfo.getBuffer());
+ written += sz;
}
}
-
+ return written;
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
index c545e7d..815536b 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
@@ -37,7 +37,7 @@
import org.apache.hyracks.dataflow.std.sort.AbstractSorterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.sort.Algorithm;
import org.apache.hyracks.dataflow.std.sort.IRunGenerator;
-import org.apache.hyracks.dataflow.std.sort.TimedRunGenerator;
+import org.apache.hyracks.dataflow.std.sort.ProfiledRunGenerator;
/**
* This Operator pushes group-by aggregation into the external sort.
@@ -148,7 +148,8 @@
recordDescriptorProvider.getInputRecordDescriptor(this.getActivityId(), 0), framesLimit,
groupFields, keyNormalizerFactories, comparatorFactories, partialAggregatorFactory,
partialAggRecordDesc, ALG);
- return profile ? TimedRunGenerator.time(runGen, ctx, "GroupBy (Sort Runs)") : runGen;
+ return profile ? ProfiledRunGenerator.time(runGen, ctx, "GroupBy (Sort Runs)", this.getActivityId())
+ : runGen;
}
};
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
index 5f80165..04f5fe8 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
@@ -34,6 +34,7 @@
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.job.profiling.IOperatorStats;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -94,6 +95,7 @@
// corresponding function signature.
private final TuplePointer tempPtr = new TuplePointer();
private int[] probePSizeInTups;
+ private IOperatorStats stats = null;
public OptimizedHybridHashJoin(IHyracksJobletContext jobletCtx, int memSizeInFrames, int numOfPartitions,
String probeRelName, String buildRelName, RecordDescriptor probeRd, RecordDescriptor buildRd,
@@ -190,7 +192,10 @@
private void spillPartition(int pid) throws HyracksDataException {
RunFileWriter writer = getSpillWriterOrCreateNewOneIfNotExist(buildRFWriters, buildRelName, pid);
- bufferManager.flushPartition(pid, writer);
+ int spilt = bufferManager.flushPartition(pid, writer);
+ if (stats != null) {
+ stats.getBytesWritten().update(spilt);
+ }
bufferManager.clearPartition(pid);
spilledStatus.set(pid);
}
@@ -268,8 +273,12 @@
for (int pid = spilledStatus.nextSetBit(0); pid >= 0 && pid < numOfPartitions; pid =
spilledStatus.nextSetBit(pid + 1)) {
if (bufferManager.getNumTuples(pid) > 0) {
- bufferManager.flushPartition(pid,
+ int spilt = bufferManager.flushPartition(pid,
getSpillWriterOrCreateNewOneIfNotExist(runFileWriters, refName, pid));
+ if (stats != null) {
+ stats.getBytesWritten().update(spilt);
+
+ }
bufferManager.clearPartition(pid);
}
}
@@ -424,6 +433,10 @@
reloadBuffer = new VSizeFrame(jobletCtx);
}
while (r.nextFrame(reloadBuffer)) {
+ if (stats != null) {
+ //TODO: be certain it is the case this is actually eagerly read
+ stats.getBytesRead().update(reloadBuffer.getBuffer().limit());
+ }
accessorBuild.reset(reloadBuffer.getBuffer());
for (int tid = 0; tid < accessorBuild.getTupleCount(); tid++) {
if (!bufferManager.insertTuple(pid, accessorBuild, tid, tempPtr)) {
@@ -527,7 +540,10 @@
if (victim >= 0 && bufferManager.getPhysicalSize(victim) >= recordSize) {
RunFileWriter runFileWriter =
getSpillWriterOrCreateNewOneIfNotExist(probeRFWriters, probeRelName, victim);
- bufferManager.flushPartition(victim, runFileWriter);
+ int spilt = bufferManager.flushPartition(victim, runFileWriter);
+ if (stats != null) {
+ stats.getBytesWritten().update(spilt);
+ }
bufferManager.clearPartition(victim);
if (!bufferManager.insertTuple(pid, accessorProbe, tupleId, tempPtr)) {
// This should not happen if the size calculations are correct, just not to let the query fail.
@@ -550,6 +566,9 @@
throw new HyracksDataException("The given tuple is too big");
}
+ if (stats != null) {
+ stats.getBytesWritten().update(bigFrameAppender.getBuffer().limit());
+ }
bigFrameAppender.write(runFileWriter, true);
}
@@ -630,4 +649,8 @@
LOGGER.debug("can't insert tuple in join memory. {}", details);
LOGGER.debug("partitions status:\n{}", spillPolicy.partitionsStatus());
}
+
+ public void setOperatorStats(IOperatorStats stats) {
+ this.stats = stats;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index 555e8fb..e04eebe 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -45,6 +45,8 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.job.profiling.IOperatorStats;
+import org.apache.hyracks.api.job.profiling.NoOpOperatorStats;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -288,6 +290,7 @@
new FieldHashPartitionComputerFamily(buildKeys, buildHashFunctionFactories)
.createPartitioner(INIT_SEED);
boolean failed = false;
+ IOperatorStats stats = new NoOpOperatorStats();
@Override
public void open() throws HyracksDataException {
@@ -300,6 +303,7 @@
state.hybridHJ = new OptimizedHybridHashJoin(ctx.getJobletContext(), state.memForJoin,
state.numOfPartitions, PROBE_REL, BUILD_REL, probeRd, buildRd, probeHpc, buildHpc,
probePredEval, buildPredEval, isLeftOuter, nonMatchWriterFactories);
+ state.hybridHJ.setOperatorStats(stats);
state.hybridHJ.initBuild();
if (LOGGER.isTraceEnabled()) {
@@ -341,6 +345,11 @@
return "Hybrid Hash Join: Build";
}
+ @Override
+ public void setOperatorStats(IOperatorStats stats) {
+ this.stats = stats;
+ }
+
};
}
}
@@ -394,6 +403,7 @@
private FrameTupleAppender nullResultAppender = null;
private FrameTupleAccessor probeTupleAccessor;
private boolean failed = false;
+ IOperatorStats stats = null;
@Override
public void open() throws HyracksDataException {
@@ -402,6 +412,7 @@
writer.open();
state.hybridHJ.initProbe(probComp);
+ state.hybridHJ.setOperatorStats(stats);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("OptimizedHybridHashJoin is starting the probe phase.");
@@ -485,6 +496,11 @@
}
}
+ @Override
+ public void setOperatorStats(IOperatorStats stats) {
+ this.stats = stats;
+ }
+
private void logProbeComplete() {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("OptimizedHybridHashJoin closed its probe phase");
@@ -505,6 +521,9 @@
long buildPartSize = (long) Math.ceil((double) buildSideReader.getFileSize() / (double) frameSize);
long probePartSize = (long) Math.ceil((double) probeSideReader.getFileSize() / (double) frameSize);
int beforeMax = Math.max(buildSizeInTuple, probeSizeInTuple);
+ if (stats.getLevel().get() < level) {
+ stats.getLevel().set(level);
+ }
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("\n>>>Joining Partition Pairs (thread_id " + Thread.currentThread().getId()
@@ -758,6 +777,7 @@
bReader.open();
rPartbuff.reset();
while (bReader.nextFrame(rPartbuff)) {
+ stats.getBytesRead().update(rPartbuff.getBuffer().limit());
// We need to allocate a copyBuffer, because this buffer gets added to the buffers list
// in the InMemoryHashJoin.
ByteBuffer copyBuffer = bufferManager.acquireFrame(rPartbuff.getFrameSize());
@@ -786,6 +806,9 @@
rPartbuff.reset();
try {
while (pReader.nextFrame(rPartbuff)) {
+ if (stats != null) {
+ stats.getBytesRead().update(rPartbuff.getBuffer().limit());
+ }
joiner.join(rPartbuff.getBuffer(), writer);
rPartbuff.reset();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
index 654f3a3..ebe871b 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
@@ -82,7 +82,8 @@
final boolean profile = ctx.getJobFlags().contains(JobFlag.PROFILE_RUNTIME);
IRunGenerator runGen = new ExternalSortRunGenerator(ctx, sortFields, keyNormalizerFactories,
comparatorFactories, outRecDescs[0], alg, policy, framesLimit, outputLimit);
- return profile ? TimedRunGenerator.time(runGen, ctx, "ExternalSort(Sort)") : runGen;
+ return profile ? ProfiledRunGenerator.time(runGen, ctx, "ExternalSort(Sort)", this.getActivityId())
+ : runGen;
}
};
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ProfiledRunGenerator.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ProfiledRunGenerator.java
new file mode 100644
index 0000000..5cc1882
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ProfiledRunGenerator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.dataflow.std.sort;
+
+import java.util.List;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.ActivityId;
+import org.apache.hyracks.api.dataflow.ProfiledFrameWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.profiling.IOperatorStats;
+import org.apache.hyracks.api.job.profiling.IStatsCollector;
+import org.apache.hyracks.api.job.profiling.OperatorStats;
+import org.apache.hyracks.dataflow.common.io.GeneratedRunFileReader;
+
+public class ProfiledRunGenerator extends ProfiledFrameWriter implements IRunGenerator {
+
+ private final IRunGenerator runGenerator;
+
+ private ProfiledRunGenerator(IRunGenerator runGenerator, IStatsCollector collector, String name,
+ IOperatorStats stats, ActivityId root) {
+ super(runGenerator, collector, name, stats, null);
+ this.runGenerator = runGenerator;
+ }
+
+ @Override
+ public List<GeneratedRunFileReader> getRuns() {
+ return runGenerator.getRuns();
+ }
+
+ @Override
+ public ISorter getSorter() {
+ return runGenerator.getSorter();
+ }
+
+ public static IRunGenerator time(IRunGenerator runGenerator, IHyracksTaskContext ctx, String name, ActivityId root)
+ throws HyracksDataException {
+ if (!(runGenerator instanceof ProfiledRunGenerator)) {
+ String statName = root.toString() + " - " + name;
+ IStatsCollector statsCollector = ctx.getStatsCollector();
+ IOperatorStats stats = new OperatorStats(statName);
+ statsCollector.add(stats);
+ return new ProfiledRunGenerator(runGenerator, ctx.getStatsCollector(), name, stats, root);
+ }
+ return runGenerator;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TimedRunGenerator.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TimedRunGenerator.java
deleted file mode 100644
index b3a4aee..0000000
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TimedRunGenerator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.dataflow.std.sort;
-
-import java.util.List;
-
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.TimedFrameWriter;
-import org.apache.hyracks.api.job.profiling.IStatsCollector;
-import org.apache.hyracks.dataflow.common.io.GeneratedRunFileReader;
-
-public class TimedRunGenerator extends TimedFrameWriter implements IRunGenerator {
-
- private final IRunGenerator runGenerator;
-
- private TimedRunGenerator(IRunGenerator runGenerator, IStatsCollector collector, String name) {
- super(runGenerator, collector, name);
- this.runGenerator = runGenerator;
- }
-
- @Override
- public List<GeneratedRunFileReader> getRuns() {
- return runGenerator.getRuns();
- }
-
- @Override
- public ISorter getSorter() {
- return runGenerator.getSorter();
- }
-
- public static IRunGenerator time(IRunGenerator runGenerator, IHyracksTaskContext ctx, String name) {
- return runGenerator instanceof TimedRunGenerator ? runGenerator
- : new TimedRunGenerator(runGenerator, ctx.getStatsCollector(), name);
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
index b7ff530..2322910 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.dataflow.common.io.GeneratedRunFileReader;
@@ -65,8 +66,13 @@
final boolean profile = ctx.getJobFlags().contains(JobFlag.PROFILE_RUNTIME);
IRunGenerator runGen = new HybridTopKSortRunGenerator(ctx, framesLimit, topK, sortFields,
keyNormalizerFactories, comparatorFactories, outRecDescs[0]);
- return profile ? TimedRunGenerator.time(runGen, ctx, "TopKSort (Sort)") : runGen;
-
+ try {
+ return profile ? ProfiledRunGenerator.time(runGen, ctx, "TopKSort (Sort)", this.getActivityId())
+ : runGen;
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ }
+ return null;
}
};
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
index efed967..6dc2eae 100644
--- a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hyracks</artifactId>
<groupId>org.apache.hyracks</groupId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-documentation/pom.xml b/hyracks-fullstack/hyracks/hyracks-documentation/pom.xml
index 624639c..bfa69fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-documentation/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-documentation/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml
index 81a3dfc..189cccf 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks.examples</groupId>
<artifactId>btree-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/pom.xml
index 7735cd7..ea327b9 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks.examples</groupId>
<artifactId>btree-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeserver/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeserver/pom.xml
index 75f4efe..31d607d 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeserver/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeserver/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks.examples</groupId>
<artifactId>btree-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/pom.xml
index 942f980..148ee48 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
index f2eb1e1..470f413 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml
index d2242c1..9dcdfa6 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/pom.xml
index 3e3bfe9..aca6991 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/pom.xml
index 0135b1a..f911fec 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml
index 57ba143..5996331 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>text-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/texthelper/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/texthelper/pom.xml
index 223e408..f91a7f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/texthelper/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/texthelper/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>text-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
index 922dfa2..dc4187a 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>text-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/pom.xml
index dadaef7..b596469 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml
index 77eedd0..2d0218c 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>tpch-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchserver/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchserver/pom.xml
index c408096..0e771b8 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchserver/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchserver/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>tpch-example</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml
index 5bf7abd..e5c1d06 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-http/pom.xml b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
index e06f6be..bdae4c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<artifactId>hyracks-http</artifactId>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
index 4732d71..1f5b441 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
@@ -43,7 +43,7 @@
//Based in part on LoggingHandler from Netty
public class CLFLogger extends ChannelDuplexHandler {
- private static final Logger accessLogger = LogManager.getLogger();
+ private static final Logger LOGGER = LogManager.getLogger();
private static final Level ACCESS_LOG_LEVEL = Level.forName("ACCESS", 550);
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withZone(ZoneId.systemDefault());
@@ -66,9 +66,14 @@
public void channelRead(ChannelHandlerContext ctx, Object msg) {
if (msg instanceof HttpRequest) {
HttpRequest req = (HttpRequest) msg;
- clientIp = ((NioSocketChannel) ctx.channel()).remoteAddress().getAddress().toString().substring(1);
+ try {
+ clientIp = ((NioSocketChannel) ctx.channel()).remoteAddress().getAddress().toString().substring(1);
+ } catch (Exception e) {
+ LOGGER.debug("ignoring {} obtaining client ip for {}", e, ctx.channel());
+ clientIp = "-";
+ }
requestTime = Instant.now();
- reqLine = req.method().toString() + " " + req.getUri() + " " + req.getProtocolVersion().toString();
+ reqLine = req.method().toString() + " " + req.uri() + " " + req.protocolVersion();
userAgentRef = headerValueOrDash("Referer", req) + " " + headerValueOrDash("User-Agent", req);
lastChunk = false;
}
@@ -116,7 +121,7 @@
}
private void printAndPrepare() {
- if (!accessLogger.isEnabled(ACCESS_LOG_LEVEL)) {
+ if (!LOGGER.isEnabled(ACCESS_LOG_LEVEL)) {
return;
}
logLineBuilder.append(clientIp);
@@ -131,7 +136,7 @@
logLineBuilder.append(" ").append(statusCode);
logLineBuilder.append(" ").append(respSize);
logLineBuilder.append(" ").append(userAgentRef);
- accessLogger.log(ACCESS_LOG_LEVEL, logLineBuilder);
+ LOGGER.log(ACCESS_LOG_LEVEL, logLineBuilder);
respSize = 0;
logLineBuilder.setLength(0);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-ipc/pom.xml b/hyracks-fullstack/hyracks/hyracks-ipc/pom.xml
index caf3f8f..ef6649b 100644
--- a/hyracks-fullstack/hyracks/hyracks-ipc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-ipc/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/pom.xml b/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/pom.xml
index fc9d0fd..2a4e635 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-maven-plugins</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml
index 16bcc7c..be23482 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-maven-plugins</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>maven-plugin</packaging>
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 5c11106..c548179 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -538,6 +538,11 @@
return;
}
Set<MavenProject> projects = new TreeSet<>(Comparator.comparing(MavenProject::getId));
+ if (!includeShadowedDependencies) {
+ getLog().info("Not gathering shadowed dependencies as 'includeShadowedDependencies' is set to "
+ + includeShadowedDependencies);
+ return;
+ }
projects.addAll(dependencyLicenseMap.keySet());
for (MavenProject p : projects) {
File artifactFile = p.getArtifact().getFile();
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/pom.xml b/hyracks-fullstack/hyracks/hyracks-maven-plugins/pom.xml
index 373aca3..eb4b086 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-net/pom.xml b/hyracks-fullstack/hyracks/hyracks-net/pom.xml
index 506eae7..ffc4e27 100644
--- a/hyracks-fullstack/hyracks/hyracks-net/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-net/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-server/pom.xml b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
index 12a723f..0c3d231 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml
index 29f1dbf..1172c8f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml
index 88b84aa..8a08074 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IDiskBTreeStatefulPointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IDiskBTreeStatefulPointSearchCursor.java
new file mode 100644
index 0000000..f436636
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IDiskBTreeStatefulPointSearchCursor.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.btree.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.impls.DiskBTreePointSearchCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+
+/**
+ * Allows stateful {@link DiskBTreePointSearchCursor} to resume the search without closing and reopening the cursor
+ * Note: upon closing a cursor, ensure the search state is cleared
+ */
+public interface IDiskBTreeStatefulPointSearchCursor {
+ int getLastPageId();
+
+ void setCursorToNextKey(ISearchPredicate searchPred) throws HyracksDataException;
+
+ ITreeIndexFrame getFrame();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
index 2455625..0ab88a5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
@@ -29,10 +29,12 @@
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
public class BTreeSearchOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
protected final int[] lowKeyFields;
protected final int[] highKeyFields;
@@ -52,6 +54,7 @@
protected byte[] searchCallbackProceedResultTrueValue;
protected final ITupleFilterFactory tupleFilterFactory;
protected final long outputLimit;
+ protected final ITupleProjectorFactory tupleProjectorFactory;
public BTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
@@ -61,7 +64,8 @@
IMissingWriterFactory nonFilterWriterFactory) {
this(spec, outRecDesc, lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, minFilterFieldIndexes,
- maxFilterFieldIndexes, appendIndexFilter, nonFilterWriterFactory, null, -1, false, null, null);
+ maxFilterFieldIndexes, appendIndexFilter, nonFilterWriterFactory, null, -1, false, null, null,
+ DefaultTupleProjectorFactory.INSTANCE);
}
public BTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
@@ -71,7 +75,7 @@
int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean appendIndexFilter,
IMissingWriterFactory nonFilterWriterFactory, ITupleFilterFactory tupleFilterFactory, long outputLimit,
boolean appendOpCallbackProceedResult, byte[] searchCallbackProceedResultFalseValue,
- byte[] searchCallbackProceedResultTrueValue) {
+ byte[] searchCallbackProceedResultTrueValue, ITupleProjectorFactory tupleProjectorFactory) {
super(spec, 1, 1);
this.indexHelperFactory = indexHelperFactory;
this.retainInput = retainInput;
@@ -92,6 +96,7 @@
this.appendOpCallbackProceedResult = appendOpCallbackProceedResult;
this.searchCallbackProceedResultFalseValue = searchCallbackProceedResultFalseValue;
this.searchCallbackProceedResultTrueValue = searchCallbackProceedResultTrueValue;
+ this.tupleProjectorFactory = tupleProjectorFactory;
}
@Override
@@ -102,7 +107,7 @@
lowKeyInclusive, highKeyInclusive, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, appendIndexFilter,
nonFilterWriterFactory, tupleFilterFactory, outputLimit, appendOpCallbackProceedResult,
- searchCallbackProceedResultFalseValue, searchCallbackProceedResultTrueValue);
+ searchCallbackProceedResultFalseValue, searchCallbackProceedResultTrueValue, tupleProjectorFactory);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index fb331bf..a56e305 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.data.accessors.PermutingFrameTupleReference;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
@@ -30,9 +31,11 @@
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
public class BTreeSearchOperatorNodePushable extends IndexSearchOperatorNodePushable {
protected final boolean lowKeyInclusive;
@@ -52,7 +55,7 @@
this(ctx, partition, inputRecDesc, lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive,
minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory, retainInput, retainMissing,
nonMatchWriterFactory, searchCallbackFactory, appendIndexFilter, nonFilterWriterFactory, null, -1,
- false, null, null);
+ false, null, null, DefaultTupleProjectorFactory.INSTANCE);
}
public BTreeSearchOperatorNodePushable(IHyracksTaskContext ctx, int partition, RecordDescriptor inputRecDesc,
@@ -62,11 +65,12 @@
ISearchOperationCallbackFactory searchCallbackFactory, boolean appendIndexFilter,
IMissingWriterFactory nonFilterWriterFactory, ITupleFilterFactory tupleFilterFactory, long outputLimit,
boolean appendOpCallbackProceedResult, byte[] searchCallbackProceedResultFalseValue,
- byte[] searchCallbackProceedResultTrueValue) throws HyracksDataException {
+ byte[] searchCallbackProceedResultTrueValue, ITupleProjectorFactory projectorFactory)
+ throws HyracksDataException {
super(ctx, inputRecDesc, partition, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, nonMatchWriterFactory, searchCallbackFactory, appendIndexFilter,
nonFilterWriterFactory, tupleFilterFactory, outputLimit, appendOpCallbackProceedResult,
- searchCallbackProceedResultFalseValue, searchCallbackProceedResultTrueValue);
+ searchCallbackProceedResultFalseValue, searchCallbackProceedResultTrueValue, projectorFactory);
this.lowKeyInclusive = lowKeyInclusive;
this.highKeyInclusive = highKeyInclusive;
if (lowKeyFields != null && lowKeyFields.length > 0) {
@@ -111,7 +115,8 @@
@Override
protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException {
- // No additional parameters are required for the B+Tree search case
+ //Set tuple projector to get the information about the pushed down value accesses (if supported by the index)
+ iap.getParameters().put(HyracksConstants.TUPLE_PROJECTOR, tupleProjector);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 78faaff..40b2f5c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -42,7 +42,6 @@
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo;
import org.apache.hyracks.storage.am.common.api.IBTreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.IPageManager;
-import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
@@ -51,7 +50,6 @@
import org.apache.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
-import org.apache.hyracks.storage.am.common.impls.NodeFrontier;
import org.apache.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
@@ -64,17 +62,13 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.NoOpIndexCursorStats;
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-import org.apache.hyracks.util.JSONUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
public class BTree extends AbstractTreeIndex {
public static final float DEFAULT_FILL_FACTOR = 0.7f;
@@ -85,7 +79,7 @@
private final AtomicInteger smoCounter;
private final ReadWriteLock treeLatch;
- private final int maxTupleSize;
+ protected final int maxTupleSize;
public BTree(IBufferCache bufferCache, IPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
@@ -886,13 +880,13 @@
}
@Override
- public BTreeRangeSearchCursor createSearchCursor(boolean exclusive) {
+ public ITreeIndexCursor createSearchCursor(boolean exclusive) {
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
return new BTreeRangeSearchCursor(leafFrame, exclusive, (IIndexCursorStats) iap.getParameters()
.getOrDefault(HyracksConstants.INDEX_CURSOR_STATS, NoOpIndexCursorStats.INSTANCE));
}
- public BTreeRangeSearchCursor createPointCursor(boolean exclusive, boolean stateful) {
+ public ITreeIndexCursor createPointCursor(boolean exclusive, boolean stateful) {
return createSearchCursor(exclusive);
}
@@ -1004,220 +998,7 @@
@Override
public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, IPageWriteCallback callback) throws HyracksDataException {
- return new BTreeBulkLoader(fillFactor, verifyInput, callback);
- }
-
- public class BTreeBulkLoader extends AbstractTreeIndex.AbstractTreeIndexBulkLoader {
- protected final ISplitKey splitKey;
- protected final boolean verifyInput;
-
- public BTreeBulkLoader(float fillFactor, boolean verifyInput, IPageWriteCallback callback)
- throws HyracksDataException {
- super(fillFactor, callback);
- this.verifyInput = verifyInput;
- splitKey = new BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
- splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
- }
-
- @Override
- public void add(ITupleReference tuple) throws HyracksDataException {
- try {
- int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple),
- interiorFrame.getBytesRequiredToWriteTuple(tuple));
- NodeFrontier leafFrontier = nodeFrontiers.get(0);
- int spaceNeeded = tupleWriter.bytesRequired(tuple) + slotSize;
- int spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace();
-
- // try to free space by compression
- if (spaceUsed + spaceNeeded > leafMaxBytes) {
- leafFrame.compress();
- spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace();
- }
- //full, allocate new page
- if (spaceUsed + spaceNeeded > leafMaxBytes) {
- if (leafFrame.getTupleCount() == 0) {
- bufferCache.returnPage(leafFrontier.page, false);
- } else {
- leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1);
- if (verifyInput) {
- verifyInputTuple(tuple, leafFrontier.lastTuple);
- }
- int splitKeySize = tupleWriter.bytesRequired(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount());
- splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount(),
- splitKey.getBuffer().array(), 0);
- splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0);
- splitKey.setLeftPage(leafFrontier.pageId);
-
- propagateBulk(1, pagesToWrite);
-
- leafFrontier.pageId = freePageManager.takePage(metaFrame);
-
- ((IBTreeLeafFrame) leafFrame).setNextLeaf(leafFrontier.pageId);
-
- write(leafFrontier.page);
- for (ICachedPage c : pagesToWrite) {
- write(c);
- }
- pagesToWrite.clear();
- splitKey.setRightPage(leafFrontier.pageId);
- }
- if (tupleSize > maxTupleSize) {
- final long dpid = BufferedFileHandle.getDiskPageId(getFileId(), leafFrontier.pageId);
- // calculate required number of pages.
- int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize());
- final int multiplier =
- (int) Math.ceil((double) tupleSize / (bufferCache.getPageSize() - headerSize));
- if (multiplier > 1) {
- leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier,
- freePageManager.takeBlock(metaFrame, multiplier - 1));
- } else {
- leafFrontier.page = bufferCache.confiscatePage(dpid);
- }
- leafFrame.setPage(leafFrontier.page);
- leafFrame.initBuffer((byte) 0);
- ((IBTreeLeafFrame) leafFrame).setLargeFlag(true);
- } else {
- final long dpid = BufferedFileHandle.getDiskPageId(getFileId(), leafFrontier.pageId);
- leafFrontier.page = bufferCache.confiscatePage(dpid);
- leafFrame.setPage(leafFrontier.page);
- leafFrame.initBuffer((byte) 0);
- }
- } else {
- if (verifyInput && leafFrame.getTupleCount() > 0) {
- leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1);
- verifyInputTuple(tuple, leafFrontier.lastTuple);
- }
- }
- ((IBTreeLeafFrame) leafFrame).insertSorted(tuple);
- } catch (HyracksDataException | RuntimeException e) {
- logState(tuple, e);
- handleException();
- throw e;
- }
- }
-
- protected void verifyInputTuple(ITupleReference tuple, ITupleReference prevTuple) throws HyracksDataException {
- // New tuple should be strictly greater than last tuple.
- int cmpResult = cmp.compare(tuple, prevTuple);
- if (cmpResult < 0) {
- throw HyracksDataException.create(ErrorCode.UNSORTED_LOAD_INPUT);
- }
- if (cmpResult == 0) {
- throw HyracksDataException.create(ErrorCode.DUPLICATE_LOAD_INPUT);
- }
- }
-
- protected void propagateBulk(int level, List<ICachedPage> pagesToWrite) throws HyracksDataException {
- if (splitKey.getBuffer() == null) {
- return;
- }
-
- if (level >= nodeFrontiers.size()) {
- addLevel();
- }
-
- NodeFrontier frontier = nodeFrontiers.get(level);
- interiorFrame.setPage(frontier.page);
-
- ITupleReference tuple = splitKey.getTuple();
- int tupleBytes = tupleWriter.bytesRequired(tuple, 0, cmp.getKeyFieldCount());
- int spaceNeeded = tupleBytes + slotSize + 4;
- if (tupleBytes > interiorFrame.getMaxTupleSize(BTree.this.bufferCache.getPageSize())) {
- throw HyracksDataException.create(ErrorCode.RECORD_IS_TOO_LARGE, tupleBytes,
- interiorFrame.getMaxTupleSize(BTree.this.bufferCache.getPageSize()));
- }
-
- int spaceUsed = interiorFrame.getBuffer().capacity() - interiorFrame.getTotalFreeSpace();
- if (spaceUsed + spaceNeeded > interiorMaxBytes) {
-
- ISplitKey copyKey = splitKey.duplicate(leafFrame.getTupleWriter().createTupleReference());
- tuple = copyKey.getTuple();
-
- frontier.lastTuple.resetByTupleIndex(interiorFrame, interiorFrame.getTupleCount() - 1);
- int splitKeySize = tupleWriter.bytesRequired(frontier.lastTuple, 0, cmp.getKeyFieldCount());
- splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(frontier.lastTuple, 0, cmp.getKeyFieldCount(),
- splitKey.getBuffer().array(), 0);
- splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0);
-
- ((IBTreeInteriorFrame) interiorFrame).deleteGreatest();
- int finalPageId = freePageManager.takePage(metaFrame);
- frontier.page.setDiskPageId(BufferedFileHandle.getDiskPageId(getFileId(), finalPageId));
- pagesToWrite.add(frontier.page);
- splitKey.setLeftPage(finalPageId);
-
- propagateBulk(level + 1, pagesToWrite);
- frontier.page = bufferCache.confiscatePage(BufferCache.INVALID_DPID);
- interiorFrame.setPage(frontier.page);
- interiorFrame.initBuffer((byte) level);
- }
- ((IBTreeInteriorFrame) interiorFrame).insertSorted(tuple);
- }
-
- private void persistFrontiers(int level, int rightPage) throws HyracksDataException {
- if (level >= nodeFrontiers.size()) {
- rootPage = nodeFrontiers.get(level - 1).pageId;
- releasedLatches = true;
- return;
- }
- if (level < 1) {
- ICachedPage lastLeaf = nodeFrontiers.get(level).page;
- int lastLeafPage = nodeFrontiers.get(level).pageId;
- lastLeaf.setDiskPageId(BufferedFileHandle.getDiskPageId(getFileId(), nodeFrontiers.get(level).pageId));
- write(lastLeaf);
- nodeFrontiers.get(level).page = null;
- persistFrontiers(level + 1, lastLeafPage);
- return;
- }
- NodeFrontier frontier = nodeFrontiers.get(level);
- interiorFrame.setPage(frontier.page);
- //just finalize = the layer right above the leaves has correct righthand pointers already
- if (rightPage < 0) {
- throw new HyracksDataException(
- "Error in index creation. Internal node appears to have no rightmost guide");
- }
- ((IBTreeInteriorFrame) interiorFrame).setRightmostChildPageId(rightPage);
- int finalPageId = freePageManager.takePage(metaFrame);
- frontier.page.setDiskPageId(BufferedFileHandle.getDiskPageId(getFileId(), finalPageId));
- write(frontier.page);
- frontier.pageId = finalPageId;
- persistFrontiers(level + 1, finalPageId);
- }
-
- @Override
- public void end() throws HyracksDataException {
- try {
- persistFrontiers(0, -1);
- super.end();
- } catch (HyracksDataException | RuntimeException e) {
- handleException();
- throw e;
- }
- }
-
- @Override
- public void abort() throws HyracksDataException {
- super.handleException();
- }
-
- private void logState(ITupleReference tuple, Exception e) {
- try {
- ObjectNode state = JSONUtil.createObject();
- state.set("leafFrame", leafFrame.getState());
- state.set("interiorFrame", interiorFrame.getState());
- int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple),
- interiorFrame.getBytesRequiredToWriteTuple(tuple));
- state.put("tupleSize", tupleSize);
- state.put("spaceNeeded", tupleWriter.bytesRequired(tuple) + slotSize);
- state.put("spaceUsed", leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace());
- state.put("leafMaxBytes", leafMaxBytes);
- state.put("maxTupleSize", maxTupleSize);
- LOGGER.error("failed to add tuple {}", state, e);
- } catch (Throwable t) {
- e.addSuppressed(t);
- }
- }
+ return new BTreeNSMBulkLoader(fillFactor, verifyInput, callback, this);
}
@SuppressWarnings("rawtypes")
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeNSMBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeNSMBulkLoader.java
new file mode 100644
index 0000000..04c84e1
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeNSMBulkLoader.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.btree.impls;
+
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
+import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import org.apache.hyracks.storage.am.common.api.ISplitKey;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndexBulkLoader;
+import org.apache.hyracks.storage.am.common.impls.NodeFrontier;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.JSONUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class BTreeNSMBulkLoader extends AbstractTreeIndexBulkLoader {
+ private static final Logger LOGGER = LogManager.getLogger();
+ protected final ISplitKey splitKey;
+ protected final boolean verifyInput;
+ private final int maxTupleSize;
+
+ public BTreeNSMBulkLoader(float fillFactor, boolean verifyInput, IPageWriteCallback callback, ITreeIndex index)
+ throws HyracksDataException {
+ this(fillFactor, verifyInput, callback, index, index.getLeafFrameFactory().createFrame());
+ }
+
+ protected BTreeNSMBulkLoader(float fillFactor, boolean verifyInput, IPageWriteCallback callback, ITreeIndex index,
+ ITreeIndexFrame leafFrame) throws HyracksDataException {
+ super(fillFactor, callback, index, leafFrame);
+ this.verifyInput = verifyInput;
+ splitKey = new BTreeSplitKey(tupleWriter.createTupleReference());
+ splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
+ maxTupleSize = ((BTree) index).maxTupleSize;
+ }
+
+ @Override
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ try {
+ int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple),
+ interiorFrame.getBytesRequiredToWriteTuple(tuple));
+ NodeFrontier leafFrontier = nodeFrontiers.get(0);
+ int spaceNeeded = tupleWriter.bytesRequired(tuple) + slotSize;
+ int spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace();
+
+ // try to free space by compression
+ if (spaceUsed + spaceNeeded > leafMaxBytes) {
+ leafFrame.compress();
+ spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace();
+ }
+ //full, allocate new page
+ if (spaceUsed + spaceNeeded > leafMaxBytes) {
+ if (leafFrame.getTupleCount() == 0) {
+ //The current page is empty. Return it.
+ bufferCache.returnPage(leafFrontier.page, false);
+ } else {
+ leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1);
+ if (verifyInput) {
+ verifyInputTuple(tuple, leafFrontier.lastTuple);
+ }
+ //The current page is not empty. Write it.
+ writeFullLeafPage();
+ }
+ if (tupleSize > maxTupleSize) {
+ //We need a large page
+ final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId);
+ // calculate required number of pages.
+ int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize());
+ final int multiplier =
+ (int) Math.ceil((double) tupleSize / (bufferCache.getPageSize() - headerSize));
+ if (multiplier > 1) {
+ leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier,
+ freePageManager.takeBlock(metaFrame, multiplier - 1));
+ } else {
+ leafFrontier.page = bufferCache.confiscatePage(dpid);
+ }
+ leafFrame.setPage(leafFrontier.page);
+ leafFrame.initBuffer((byte) 0);
+ ((IBTreeLeafFrame) leafFrame).setLargeFlag(true);
+ } else {
+ //allocate a new page
+ confiscateNewLeafPage();
+ }
+ } else {
+ if (verifyInput && leafFrame.getTupleCount() > 0) {
+ leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1);
+ verifyInputTuple(tuple, leafFrontier.lastTuple);
+ }
+ }
+ ((IBTreeLeafFrame) leafFrame).insertSorted(tuple);
+ } catch (HyracksDataException | RuntimeException e) {
+ logState(tuple, e);
+ handleException();
+ throw e;
+ }
+ }
+
+ protected void verifyInputTuple(ITupleReference tuple, ITupleReference prevTuple) throws HyracksDataException {
+ // New tuple should be strictly greater than last tuple.
+ int cmpResult = cmp.compare(tuple, prevTuple);
+ if (cmpResult < 0) {
+ throw HyracksDataException.create(ErrorCode.UNSORTED_LOAD_INPUT);
+ }
+ if (cmpResult == 0) {
+ throw HyracksDataException.create(ErrorCode.DUPLICATE_LOAD_INPUT);
+ }
+ }
+
+ protected void propagateBulk(int level, List<ICachedPage> pagesToWrite) throws HyracksDataException {
+ if (splitKey.getBuffer() == null) {
+ return;
+ }
+
+ if (level >= nodeFrontiers.size()) {
+ addLevel();
+ }
+
+ NodeFrontier frontier = nodeFrontiers.get(level);
+ interiorFrame.setPage(frontier.page);
+
+ ITupleReference tuple = splitKey.getTuple();
+ int tupleBytes = tupleWriter.bytesRequired(tuple, 0, cmp.getKeyFieldCount());
+ int spaceNeeded = tupleBytes + slotSize + 4;
+ if (tupleBytes > interiorFrame.getMaxTupleSize(bufferCache.getPageSize())) {
+ throw HyracksDataException.create(ErrorCode.RECORD_IS_TOO_LARGE, tupleBytes,
+ interiorFrame.getMaxTupleSize(bufferCache.getPageSize()));
+ }
+
+ int spaceUsed = interiorFrame.getBuffer().capacity() - interiorFrame.getTotalFreeSpace();
+ if (spaceUsed + spaceNeeded > interiorMaxBytes) {
+ ISplitKey copyKey = splitKey.duplicate(tupleWriter.createTupleReference());
+ tuple = copyKey.getTuple();
+
+ frontier.lastTuple.resetByTupleIndex(interiorFrame, interiorFrame.getTupleCount() - 1);
+ int splitKeySize = tupleWriter.bytesRequired(frontier.lastTuple, 0, cmp.getKeyFieldCount());
+ splitKey.initData(splitKeySize);
+ tupleWriter.writeTupleFields(frontier.lastTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer().array(),
+ 0);
+ splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0);
+
+ ((IBTreeInteriorFrame) interiorFrame).deleteGreatest();
+ int finalPageId = freePageManager.takePage(metaFrame);
+ frontier.page.setDiskPageId(BufferedFileHandle.getDiskPageId(fileId, finalPageId));
+ pagesToWrite.add(frontier.page);
+ splitKey.setLeftPage(finalPageId);
+
+ propagateBulk(level + 1, pagesToWrite);
+ frontier.page = bufferCache.confiscatePage(IBufferCache.INVALID_DPID);
+ interiorFrame.setPage(frontier.page);
+ interiorFrame.initBuffer((byte) level);
+ }
+ ((IBTreeInteriorFrame) interiorFrame).insertSorted(tuple);
+ }
+
+ private void persistFrontiers(int level, int rightPage) throws HyracksDataException {
+ if (level >= nodeFrontiers.size()) {
+ setRootPageId(nodeFrontiers.get(level - 1).pageId);
+ releasedLatches = true;
+ return;
+ }
+ if (level < 1) {
+ ICachedPage lastLeaf = nodeFrontiers.get(level).page;
+ int lastLeafPage = nodeFrontiers.get(level).pageId;
+ lastLeaf.setDiskPageId(BufferedFileHandle.getDiskPageId(fileId, nodeFrontiers.get(level).pageId));
+ writeLastLeaf(lastLeaf);
+ nodeFrontiers.get(level).page = null;
+ persistFrontiers(level + 1, lastLeafPage);
+ return;
+ }
+ NodeFrontier frontier = nodeFrontiers.get(level);
+ interiorFrame.setPage(frontier.page);
+ //just finalize = the layer right above the leaves has correct righthand pointers already
+ if (rightPage < 0) {
+ throw new HyracksDataException("Error in index creation. Internal node appears to have no rightmost guide");
+ }
+ ((IBTreeInteriorFrame) interiorFrame).setRightmostChildPageId(rightPage);
+ int finalPageId = freePageManager.takePage(metaFrame);
+ frontier.page.setDiskPageId(BufferedFileHandle.getDiskPageId(fileId, finalPageId));
+ write(frontier.page);
+ frontier.pageId = finalPageId;
+ persistFrontiers(level + 1, finalPageId);
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ try {
+ persistFrontiers(0, -1);
+ super.end();
+ } catch (HyracksDataException | RuntimeException e) {
+ handleException();
+ throw e;
+ }
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ super.handleException();
+ }
+
+ protected void writeFullLeafPage() throws HyracksDataException {
+ final NodeFrontier leafFrontier = nodeFrontiers.get(0);
+ leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1);
+ final int splitKeySize = tupleWriter.bytesRequired(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount());
+ splitKey.initData(splitKeySize);
+ tupleWriter.writeTupleFields(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer().array(),
+ 0);
+ splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0);
+ splitKey.setLeftPage(leafFrontier.pageId);
+
+ propagateBulk(1, pagesToWrite);
+
+ leafFrontier.pageId = freePageManager.takePage(metaFrame);
+
+ ((IBTreeLeafFrame) leafFrame).setNextLeaf(leafFrontier.pageId);
+
+ write(leafFrontier.page);
+ for (ICachedPage c : pagesToWrite) {
+ write(c);
+ }
+ pagesToWrite.clear();
+ splitKey.setRightPage(leafFrontier.pageId);
+ }
+
+ protected void writeLastLeaf(ICachedPage page) throws HyracksDataException {
+ write(page);
+ }
+
+ protected final void confiscateNewLeafPage() throws HyracksDataException {
+ final NodeFrontier leafFrontier = nodeFrontiers.get(0);
+ final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId);
+ leafFrontier.page = bufferCache.confiscatePage(dpid);
+ leafFrame.setPage(leafFrontier.page);
+ leafFrame.initBuffer((byte) 0);
+ }
+
+ private void logState(ITupleReference tuple, Exception e) {
+ try {
+ ObjectNode state = JSONUtil.createObject();
+ state.set("leafFrame", leafFrame.getState());
+ state.set("interiorFrame", interiorFrame.getState());
+ int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple),
+ interiorFrame.getBytesRequiredToWriteTuple(tuple));
+ state.put("tupleSize", tupleSize);
+ state.put("spaceNeeded", tupleWriter.bytesRequired(tuple) + slotSize);
+ state.put("spaceUsed", leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace());
+ state.put("leafMaxBytes", leafMaxBytes);
+ state.put("maxTupleSize", maxTupleSize);
+ LOGGER.error("failed to add tuple {}", state, e);
+ } catch (Throwable t) {
+ e.addSuppressed(t);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java
index ae6bbaa..179f1da 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java
@@ -24,8 +24,8 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.btree.api.IBTreeFrame;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import org.apache.hyracks.storage.am.btree.api.IDiskBTreeStatefulPointSearchCursor;
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
@@ -38,7 +38,6 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.NoOpIndexCursorStats;
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
@@ -76,7 +75,8 @@
private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, BTreeOpContext ctx)
throws HyracksDataException {
ctx.reset();
- ctx.setPred((RangePredicate) searchPred);
+ RangePredicate rangePredicate = (RangePredicate) searchPred;
+ ctx.setPred(rangePredicate);
ctx.setCursor(cursor);
if (ctx.getPred().getLowKeyComparator() == null) {
ctx.getPred().setLowKeyComparator(ctx.getCmp());
@@ -87,25 +87,17 @@
cursor.setBufferCache(bufferCache);
cursor.setFileId(getFileId());
- if (cursor instanceof DiskBTreePointSearchCursor) {
- DiskBTreePointSearchCursor pointCursor = (DiskBTreePointSearchCursor) cursor;
+ if (cursor instanceof IDiskBTreeStatefulPointSearchCursor) {
+ IDiskBTreeStatefulPointSearchCursor pointCursor = (IDiskBTreeStatefulPointSearchCursor) cursor;
int lastPageId = pointCursor.getLastPageId();
- if (lastPageId != BufferCache.INVALID_PAGEID) {
- // check whether the last leaf page contains this key
- ICachedPage lastPage =
- bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), lastPageId), false);
- ctx.getLeafFrame().setPage(lastPage);
- if (fitInPage(ctx.getPred().getLowKey(), ctx.getPred().getLowKeyComparator(), ctx.getLeafFrame())) {
- // use this page
- ctx.getCursorInitialState().setPage(lastPage);
- ctx.getCursorInitialState().setPageId(lastPageId);
- pointCursor.open(ctx.getCursorInitialState(), searchPred);
+ if (lastPageId != IBufferCache.INVALID_PAGEID) {
+ if (fitInPage(ctx.getPred().getLowKey(), ctx.getPred().getLowKeyComparator(), pointCursor.getFrame())) {
+ pointCursor.setCursorToNextKey(searchPred);
return;
} else {
- // release the last page and clear the states of this cursor
+ // release the last page, clear the states of this cursor, and close the cursor
// then retry the search from root to leaf
- bufferCache.unpin(lastPage);
- pointCursor.clearSearchState();
+ cursor.close();
}
}
}
@@ -113,7 +105,7 @@
searchDown(rootNode, rootPage, ctx, cursor);
}
- private boolean fitInPage(ITupleReference key, MultiComparator comparator, IBTreeFrame frame)
+ private boolean fitInPage(ITupleReference key, MultiComparator comparator, ITreeIndexFrame frame)
throws HyracksDataException {
// assume that search keys are sorted (non-decreasing)
ITupleReference rightmostTuple = frame.getRightmostTuple();
@@ -191,14 +183,14 @@
}
@Override
- public DiskBTreeRangeSearchCursor createSearchCursor(boolean exclusive) {
+ public ITreeIndexCursor createSearchCursor(boolean exclusive) {
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
return new DiskBTreeRangeSearchCursor(leafFrame, exclusive, (IIndexCursorStats) iap.getParameters()
.getOrDefault(HyracksConstants.INDEX_CURSOR_STATS, NoOpIndexCursorStats.INSTANCE));
}
@Override
- public BTreeRangeSearchCursor createPointCursor(boolean exclusive, boolean stateful) {
+ public ITreeIndexCursor createPointCursor(boolean exclusive, boolean stateful) {
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
return new DiskBTreePointSearchCursor(leafFrame, exclusive, stateful);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreePointSearchCursor.java
index 1bf3ecf..8fd9a96 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreePointSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreePointSearchCursor.java
@@ -21,13 +21,16 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import org.apache.hyracks.storage.am.btree.api.IDiskBTreeStatefulPointSearchCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode;
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-public class DiskBTreePointSearchCursor extends DiskBTreeRangeSearchCursor {
+public class DiskBTreePointSearchCursor extends DiskBTreeRangeSearchCursor
+ implements IDiskBTreeStatefulPointSearchCursor {
/**
* A stateful cursor keeps the search state (last search page Id + index) across multiple searches
* until {@link #clearSearchState()} is called explicity
@@ -36,16 +39,14 @@
private boolean nextHasBeenCalled;
- private int lastPageId = BufferCache.INVALID_PAGEID;
+ private int lastPageId;
private int lastTupleIndex = 0;
public DiskBTreePointSearchCursor(IBTreeLeafFrame frame, boolean exclusiveLatchNodes, boolean stateful) {
super(frame, exclusiveLatchNodes);
this.stateful = stateful;
- }
-
- public DiskBTreePointSearchCursor(IBTreeLeafFrame frame, boolean exclusiveLatchNodes) {
- this(frame, exclusiveLatchNodes, false);
+ lastPageId = IBufferCache.INVALID_PAGEID;
+ lastTupleIndex = 0;
}
@Override
@@ -59,6 +60,12 @@
}
@Override
+ public void doClose() throws HyracksDataException {
+ clearSearchState();
+ super.doClose();
+ }
+
+ @Override
public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
// in case open is called multiple times without closing
if (page != null) {
@@ -71,30 +78,10 @@
page = initialState.getPage();
isPageDirty = false;
frame.setPage(page);
-
- pred = (RangePredicate) searchPred;
- lowKeyCmp = pred.getLowKeyComparator();
- lowKey = pred.getLowKey();
-
- reusablePredicate.setLowKeyComparator(originalKeyCmp);
-
- lowKeyFtm = FindTupleMode.EXACT;
- lowKeyFtp = FindTupleNoExactMatchPolicy.NONE;
-
- nextHasBeenCalled = false;
-
- // only get the low key position
- tupleIndex = getLowKeyIndex();
- if (stateful) {
- lastPageId = pageId;
- if (tupleIndex >= 0) {
- lastTupleIndex = tupleIndex;
- } else {
- lastTupleIndex = -tupleIndex - 1;
- }
- }
+ setCursorToNextKey(searchPred);
}
+ @Override
public int getLastPageId() {
return lastPageId;
}
@@ -108,9 +95,38 @@
}
}
- public void clearSearchState() {
- this.lastPageId = BufferCache.INVALID_PAGEID;
+ @Override
+ public void setCursorToNextKey(ISearchPredicate searchPred) throws HyracksDataException {
+ pred = (RangePredicate) searchPred;
+ lowKeyCmp = pred.getLowKeyComparator();
+ lowKey = pred.getLowKey();
+
+ reusablePredicate.setLowKeyComparator(originalKeyCmp);
+
+ lowKeyFtm = FindTupleMode.EXACT;
+ lowKeyFtp = FindTupleNoExactMatchPolicy.NONE;
+
+ nextHasBeenCalled = false;
+
+ // only get the lowKey position
+ tupleIndex = getLowKeyIndex();
+ if (stateful) {
+ lastPageId = pageId;
+ if (tupleIndex >= 0) {
+ lastTupleIndex = tupleIndex;
+ } else {
+ lastTupleIndex = -tupleIndex - 1;
+ }
+ }
+ }
+
+ private void clearSearchState() {
+ this.lastPageId = IBufferCache.INVALID_PAGEID;
this.lastTupleIndex = 0;
}
+ @Override
+ public ITreeIndexFrame getFrame() {
+ return frame;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml
index 82937cf..9d869e9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index eab8c96..4fc8057 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -24,11 +24,13 @@
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IIntrospectingOperator;
import org.apache.hyracks.api.dataflow.value.IMissingWriter;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.profiling.IOperatorStats;
+import org.apache.hyracks.api.job.profiling.NoOpOperatorStats;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.api.util.ExceptionUtils;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -44,6 +46,7 @@
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilter;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.tuples.ReferenceFrameTupleReference;
@@ -54,12 +57,15 @@
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.projection.ITupleProjector;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
import org.apache.hyracks.util.IThreadStatsCollector;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
+public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable
+ implements IIntrospectingOperator {
static final Logger LOGGER = LogManager.getLogger();
protected final IHyracksTaskContext ctx;
@@ -107,6 +113,7 @@
protected final long outputLimit;
protected long outputCount = 0;
protected boolean finished;
+ protected final ITupleProjector tupleProjector;
// no filter and limit pushdown
public IndexSearchOperatorNodePushable(IHyracksTaskContext ctx, RecordDescriptor inputRecDesc, int partition,
@@ -116,7 +123,7 @@
IMissingWriterFactory nonFilterWriterFactory) throws HyracksDataException {
this(ctx, inputRecDesc, partition, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, nonMatchWriterFactory, searchCallbackFactory, appendIndexFilter,
- nonFilterWriterFactory, null, -1, false, null, null);
+ nonFilterWriterFactory, null, -1, false, null, null, DefaultTupleProjectorFactory.INSTANCE);
}
public IndexSearchOperatorNodePushable(IHyracksTaskContext ctx, RecordDescriptor inputRecDesc, int partition,
@@ -125,7 +132,8 @@
ISearchOperationCallbackFactory searchCallbackFactory, boolean appendIndexFilter,
IMissingWriterFactory nonFilterWriterFactory, ITupleFilterFactory tupleFactoryFactory, long outputLimit,
boolean appendSearchCallbackProceedResult, byte[] searchCallbackProceedResultFalseValue,
- byte[] searchCallbackProceedResultTrueValue) throws HyracksDataException {
+ byte[] searchCallbackProceedResultTrueValue, ITupleProjectorFactory projectorFactory)
+ throws HyracksDataException {
this.ctx = ctx;
this.indexHelper = indexHelperFactory.create(ctx.getJobletContext().getServiceContext(), partition);
this.retainInput = retainInput;
@@ -154,14 +162,13 @@
this.searchCallbackProceedResultTrueValue = searchCallbackProceedResultTrueValue;
this.tupleFilterFactory = tupleFactoryFactory;
this.outputLimit = outputLimit;
-
- if (ctx != null && ctx.getStatsCollector() != null) {
- stats = ctx.getStatsCollector().getOrAddOperatorStats(getDisplayName());
- }
+ this.stats = new NoOpOperatorStats();
if (this.tupleFilterFactory != null && this.retainMissing) {
throw new IllegalStateException("RetainMissing with tuple filter is not supported");
}
+
+ tupleProjector = projectorFactory.createTupleProjector(ctx);
}
protected abstract ISearchPredicate createSearchPredicate();
@@ -236,12 +243,6 @@
cursor.next();
matchingTupleCount++;
ITupleReference tuple = cursor.getTuple();
- if (tupleFilter != null) {
- referenceFilterTuple.reset(tuple);
- if (!tupleFilter.accept(referenceFilterTuple)) {
- continue;
- }
- }
tb.reset();
if (retainInput) {
@@ -251,7 +252,17 @@
tb.addFieldEndOffset();
}
}
- writeTupleToOutput(tuple);
+
+ // tuple must be written first before the filter is applied to
+ // assemble columnar tuples
+ tuple = writeTupleToOutput(tuple);
+ if (tupleFilter != null) {
+ referenceFilterTuple.reset(tuple);
+ if (!tupleFilter.accept(referenceFilterTuple)) {
+ continue;
+ }
+ }
+
if (appendSearchCallbackProceedResult) {
writeSearchCallbackProceedResult(tb,
((ILSMIndexCursor) cursor).getSearchOperationCallbackProceedResult());
@@ -266,7 +277,7 @@
break;
}
}
- stats.getTupleCounter().update(matchingTupleCount);
+ stats.getInputTupleCounter().update(matchingTupleCount);
if (matchingTupleCount == 0 && retainInput && retainMissing) {
FrameUtils.appendConcatToWriter(writer, appender, accessor, tupleIndex,
@@ -314,7 +325,8 @@
if (appender.getTupleCount() > 0) {
appender.write(writer, true);
}
- stats.getDiskIoCounter().update(ctx.getThreadStats().getPinnedPagesCount());
+ stats.getPageReads().update(ctx.getThreadStats().getPinnedPagesCount());
+ stats.coldReadCounter().update(ctx.getThreadStats().getColdReadCount());
} catch (Throwable th) { // NOSONAR Must ensure writer.fail is called.
// subsequently, the failure will be thrown
failure = th;
@@ -347,12 +359,9 @@
}
}
- protected void writeTupleToOutput(ITupleReference tuple) throws IOException {
+ protected ITupleReference writeTupleToOutput(ITupleReference tuple) throws IOException {
try {
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
- tb.addFieldEndOffset();
- }
+ return tupleProjector.project(tuple, dos, tb);
} catch (Exception e) {
throw e;
}
@@ -399,4 +408,9 @@
return "Index Search";
}
+ @Override
+ public void setOperatorStats(IOperatorStats stats) {
+ this.stats = stats;
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 81e528b..11368bf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -19,9 +19,6 @@
package org.apache.hyracks.storage.am.common.impls;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -30,17 +27,9 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import org.apache.hyracks.storage.common.IIndexBulkLoader;
-import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.HaltOnFailureCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.buffercache.IFIFOPageWriter;
-import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
-import org.apache.hyracks.storage.common.buffercache.PageWriteFailureCallback;
-import org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
public abstract class AbstractTreeIndex implements ITreeIndex {
@@ -229,118 +218,6 @@
return fieldCount;
}
- public abstract class AbstractTreeIndexBulkLoader extends PageWriteFailureCallback implements IIndexBulkLoader {
- protected final MultiComparator cmp;
- protected final int slotSize;
- protected final int leafMaxBytes;
- protected final int interiorMaxBytes;
- protected final ArrayList<NodeFrontier> nodeFrontiers = new ArrayList<>();
- protected final ITreeIndexMetadataFrame metaFrame;
- protected final ITreeIndexTupleWriter tupleWriter;
- protected ITreeIndexFrame leafFrame;
- protected ITreeIndexFrame interiorFrame;
- // Immutable bulk loaders write their root page at page -2, as needed e.g. by append-only file systems such as
- // HDFS. Since loading this tree relies on the root page actually being at that point, no further inserts into
- // that tree are allowed. Currently, this is not enforced.
- protected boolean releasedLatches;
- private final IFIFOPageWriter pageWriter;
- protected List<ICachedPage> pagesToWrite;
- private final ICompressedPageWriter compressedPageWriter;
-
- public AbstractTreeIndexBulkLoader(float fillFactor, IPageWriteCallback callback) throws HyracksDataException {
- leafFrame = leafFrameFactory.createFrame();
- interiorFrame = interiorFrameFactory.createFrame();
- metaFrame = freePageManager.createMetadataFrame();
-
- pageWriter = bufferCache.createFIFOWriter(callback, this);
-
- if (!isEmptyTree(leafFrame)) {
- throw HyracksDataException.create(ErrorCode.CANNOT_BULK_LOAD_NON_EMPTY_TREE);
- }
-
- this.cmp = MultiComparator.create(cmpFactories);
-
- leafFrame.setMultiComparator(cmp);
- interiorFrame.setMultiComparator(cmp);
-
- tupleWriter = leafFrame.getTupleWriter();
-
- NodeFrontier leafFrontier = new NodeFrontier(leafFrame.createTupleReference());
- leafFrontier.pageId = freePageManager.takePage(metaFrame);
- leafFrontier.page =
- bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId));
-
- interiorFrame.setPage(leafFrontier.page);
- interiorFrame.initBuffer((byte) 0);
- interiorMaxBytes = (int) (interiorFrame.getBuffer().capacity() * fillFactor);
-
- leafFrame.setPage(leafFrontier.page);
- leafFrame.initBuffer((byte) 0);
- leafMaxBytes = (int) (leafFrame.getBuffer().capacity() * fillFactor);
- slotSize = leafFrame.getSlotSize();
-
- nodeFrontiers.add(leafFrontier);
- pagesToWrite = new ArrayList<>();
- compressedPageWriter = bufferCache.getCompressedPageWriter(fileId);
- }
-
- protected void handleException() {
- // Unlatch and unpin pages that weren't in the queue to avoid leaking memory.
- compressedPageWriter.abort();
- for (NodeFrontier nodeFrontier : nodeFrontiers) {
- if (nodeFrontier != null && nodeFrontier.page != null) {
- ICachedPage frontierPage = nodeFrontier.page;
- if (frontierPage.confiscated()) {
- bufferCache.returnPage(frontierPage, false);
- }
- }
- }
- for (ICachedPage pageToDiscard : pagesToWrite) {
- if (pageToDiscard != null) {
- bufferCache.returnPage(pageToDiscard, false);
- }
- }
- releasedLatches = true;
- }
-
- @Override
- public void end() throws HyracksDataException {
- if (hasFailed()) {
- throw HyracksDataException.create(getFailure());
- }
- freePageManager.setRootPageId(rootPage);
- }
-
- protected void addLevel() throws HyracksDataException {
- NodeFrontier frontier = new NodeFrontier(tupleWriter.createTupleReference());
- frontier.page = bufferCache.confiscatePage(IBufferCache.INVALID_DPID);
- frontier.pageId = -1;
- frontier.lastTuple.setFieldCount(cmp.getKeyFieldCount());
- interiorFrame.setPage(frontier.page);
- interiorFrame.initBuffer((byte) nodeFrontiers.size());
- nodeFrontiers.add(frontier);
- }
-
- public ITreeIndexFrame getLeafFrame() {
- return leafFrame;
- }
-
- public void setLeafFrame(ITreeIndexFrame leafFrame) {
- this.leafFrame = leafFrame;
- }
-
- public void write(ICachedPage cPage) throws HyracksDataException {
- compressedPageWriter.prepareWrite(cPage);
- pageWriter.write(cPage);
- }
-
- @Override
- public void force() throws HyracksDataException {
- bufferCache.force(fileId, false);
- }
-
- }
-
public IBinaryComparatorFactory[] getCmpFactories() {
return cmpFactories;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndexBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndexBulkLoader.java
new file mode 100644
index 0000000..45a88a7
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndexBulkLoader.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.impls;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IFIFOPageWriter;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.buffercache.PageWriteFailureCallback;
+import org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+public abstract class AbstractTreeIndexBulkLoader extends PageWriteFailureCallback implements IIndexBulkLoader {
+ protected final IBufferCache bufferCache;
+ protected final IPageManager freePageManager;
+ protected final AbstractTreeIndex treeIndex;
+ protected final int fileId;
+ protected final MultiComparator cmp;
+ protected final int slotSize;
+ protected final int leafMaxBytes;
+ protected final int interiorMaxBytes;
+ protected final ArrayList<NodeFrontier> nodeFrontiers = new ArrayList<>();
+ protected final ITreeIndexMetadataFrame metaFrame;
+ protected final ITreeIndexTupleWriter tupleWriter;
+ protected ITreeIndexFrame leafFrame;
+ protected ITreeIndexFrame interiorFrame;
+ // Immutable bulk loaders write their root page at page -2, as needed e.g. by append-only file systems such as
+ // HDFS. Since loading this tree relies on the root page actually being at that point, no further inserts into
+ // that tree are allowed. Currently, this is not enforced.
+ protected boolean releasedLatches;
+ private final IFIFOPageWriter pageWriter;
+ protected List<ICachedPage> pagesToWrite;
+ private final ICompressedPageWriter compressedPageWriter;
+
+ protected AbstractTreeIndexBulkLoader(float fillFactor, IPageWriteCallback callback, ITreeIndex index)
+ throws HyracksDataException {
+ this(fillFactor, callback, index, index.getLeafFrameFactory().createFrame());
+ }
+
+ protected AbstractTreeIndexBulkLoader(float fillFactor, IPageWriteCallback callback, ITreeIndex index,
+ ITreeIndexFrame leafFrame) throws HyracksDataException {
+ this.bufferCache = index.getBufferCache();
+ this.freePageManager = index.getPageManager();
+ this.fileId = index.getFileId();
+ this.treeIndex = (AbstractTreeIndex) index;
+ this.leafFrame = leafFrame;
+ interiorFrame = treeIndex.getInteriorFrameFactory().createFrame();
+ metaFrame = freePageManager.createMetadataFrame();
+
+ pageWriter = bufferCache.createFIFOWriter(callback, this);
+
+ if (!treeIndex.isEmptyTree(leafFrame)) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_BULK_LOAD_NON_EMPTY_TREE);
+ }
+
+ this.cmp = MultiComparator.create(treeIndex.getCmpFactories());
+
+ leafFrame.setMultiComparator(cmp);
+ interiorFrame.setMultiComparator(cmp);
+
+ tupleWriter = leafFrame.getTupleWriter();
+ NodeFrontier leafFrontier = new NodeFrontier(createTupleReference());
+ leafFrontier.pageId = freePageManager.takePage(metaFrame);
+ leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId));
+
+ interiorFrame.setPage(leafFrontier.page);
+ interiorFrame.initBuffer((byte) 0);
+ interiorMaxBytes = (int) (interiorFrame.getBuffer().capacity() * fillFactor);
+
+ leafFrame.setPage(leafFrontier.page);
+ leafFrame.initBuffer((byte) 0);
+ leafMaxBytes = (int) (leafFrame.getBuffer().capacity() * fillFactor);
+ slotSize = leafFrame.getSlotSize();
+
+ nodeFrontiers.add(leafFrontier);
+ pagesToWrite = new ArrayList<>();
+ compressedPageWriter = bufferCache.getCompressedPageWriter(fileId);
+ }
+
+ protected ITreeIndexTupleReference createTupleReference() {
+ return leafFrame.createTupleReference();
+ }
+
+ protected void handleException() {
+ // Unlatch and unpin pages that weren't in the queue to avoid leaking memory.
+ compressedPageWriter.abort();
+ for (NodeFrontier nodeFrontier : nodeFrontiers) {
+ if (nodeFrontier != null && nodeFrontier.page != null) {
+ ICachedPage frontierPage = nodeFrontier.page;
+ if (frontierPage.confiscated()) {
+ bufferCache.returnPage(frontierPage, false);
+ }
+ }
+ }
+ for (ICachedPage pageToDiscard : pagesToWrite) {
+ if (pageToDiscard != null) {
+ bufferCache.returnPage(pageToDiscard, false);
+ }
+ }
+ releasedLatches = true;
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ if (hasFailed()) {
+ throw HyracksDataException.create(getFailure());
+ }
+ freePageManager.setRootPageId(treeIndex.getRootPageId());
+ }
+
+ protected void setRootPageId(int rootPage) {
+ treeIndex.rootPage = rootPage;
+ }
+
+ protected void addLevel() throws HyracksDataException {
+ NodeFrontier frontier = new NodeFrontier(tupleWriter.createTupleReference());
+ frontier.page = bufferCache.confiscatePage(IBufferCache.INVALID_DPID);
+ frontier.pageId = -1;
+ frontier.lastTuple.setFieldCount(cmp.getKeyFieldCount());
+ interiorFrame.setPage(frontier.page);
+ interiorFrame.initBuffer((byte) nodeFrontiers.size());
+ nodeFrontiers.add(frontier);
+ }
+
+ public ITreeIndexFrame getLeafFrame() {
+ return leafFrame;
+ }
+
+ public void setLeafFrame(ITreeIndexFrame leafFrame) {
+ this.leafFrame = leafFrame;
+ }
+
+ public void write(ICachedPage cPage) throws HyracksDataException {
+ compressedPageWriter.prepareWrite(cPage);
+ pageWriter.write(cPage);
+ }
+
+ @Override
+ public void force() throws HyracksDataException {
+ bufferCache.force(fileId, false);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/DefaultTupleProjector.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/DefaultTupleProjector.java
new file mode 100644
index 0000000..c63912b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/DefaultTupleProjector.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.impls;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.common.projection.ITupleProjector;
+
+class DefaultTupleProjector implements ITupleProjector {
+ public static final ITupleProjector INSTANCE = new DefaultTupleProjector();
+
+ private DefaultTupleProjector() {
+ }
+
+ @Override
+ public ITupleReference project(ITupleReference tuple, DataOutput dos, ArrayTupleBuilder tb) throws IOException {
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ return tuple;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/DefaultTupleProjectorFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/DefaultTupleProjectorFactory.java
new file mode 100644
index 0000000..092982d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/DefaultTupleProjectorFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.impls;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.projection.ITupleProjector;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
+
+public class DefaultTupleProjectorFactory implements ITupleProjectorFactory {
+ private static final long serialVersionUID = -4525893018744087821L;
+ public static final DefaultTupleProjectorFactory INSTANCE = new DefaultTupleProjectorFactory();
+
+ private DefaultTupleProjectorFactory() {
+ }
+
+ @Override
+ public ITupleProjector createTupleProjector(IHyracksTaskContext context) throws HyracksDataException {
+ return DefaultTupleProjector.INSTANCE;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/IndexAccessParameters.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/IndexAccessParameters.java
index 11d3cd5..63902be 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/IndexAccessParameters.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/IndexAccessParameters.java
@@ -59,6 +59,11 @@
return paramMap;
}
+ @Override
+ public <T> T getParameter(String key, Class<T> clazz) {
+ return paramMap != null && clazz.isInstance(paramMap.get(key)) ? clazz.cast(paramMap.get(key)) : null;
+ }
+
public static IIndexAccessParameters createNoOpParams(IIndexCursorStats stats) {
if (stats == NoOpIndexCursorStats.INSTANCE) {
return NoOpIndexAccessParameters.INSTANCE;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
index fa7811c..76a1930 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
@@ -47,4 +47,9 @@
public Map<String, Object> getParameters() {
return paramMap;
}
+
+ @Override
+ public <T> T getParameter(String key, Class<T> clazz) {
+ return null;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/pom.xml
new file mode 100644
index 0000000..8e0bc0c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/pom.xml
@@ -0,0 +1,99 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-storage-am-lsm-btree-column</artifactId>
+ <parent>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.3.9-SNAPSHOT</version>
+ </parent>
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+ <properties>
+ <root.dir>${basedir}/../..</root.dir>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-bloomfilter</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReader.java
new file mode 100644
index 0000000..7db792b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+
+/**
+ * Provided for columnar read tuple reference
+ */
+public abstract class AbstractColumnTupleReader extends AbstractTupleWriterDisabledMethods {
+ public abstract IColumnTupleIterator createTupleIterator(ColumnBTreeReadLeafFrame frame, int componentIndex,
+ IColumnReadMultiPageOp multiPageOp);
+
+ /**
+ * Currently fixed to 4-byte per offset
+ *
+ * @param buf buffer of Page0
+ * @param columnIndex column index
+ * @return column offset
+ * @see AbstractColumnTupleWriter#getColumnOffsetsSize()
+ */
+ public final int getColumnOffset(ByteBuffer buf, int columnIndex) {
+ return buf.getInt(AbstractColumnBTreeLeafFrame.HEADER_SIZE + columnIndex * Integer.BYTES);
+ }
+
+ @Override
+ public final int bytesRequired(ITupleReference tuple) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReaderWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReaderWriterFactory.java
new file mode 100644
index 0000000..774bbb9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReaderWriterFactory.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+/**
+ * For columns, there are two types for {@link ITreeIndexTupleWriter} one used during write and another during read
+ */
+public abstract class AbstractColumnTupleReaderWriterFactory implements ITreeIndexTupleWriterFactory {
+ private static final long serialVersionUID = -2377235465942457248L;
+ protected final int pageSize;
+ protected final int maxNumberOfTuples;
+ protected final float tolerance;
+
+ /**
+ * Tuple reader/writer factory
+ *
+ * @param pageSize {@link IBufferCache} page size
+ * @param maxNumberOfTuples maximum number of tuples stored per a mega leaf page
+ * @param tolerance percentage of tolerated empty space
+ */
+ protected AbstractColumnTupleReaderWriterFactory(int pageSize, int maxNumberOfTuples, float tolerance) {
+ this.pageSize = pageSize;
+ this.maxNumberOfTuples = maxNumberOfTuples;
+ this.tolerance = tolerance;
+ }
+
+ /**
+ * Create columnar tuple writer
+ *
+ * @param columnMetadata writer column metadata
+ */
+ public abstract AbstractColumnTupleWriter createColumnWriter(IColumnMetadata columnMetadata);
+
+ /**
+ * Create columnar tuple reader
+ *
+ * @param columnProjectionInfo column projection info for either query or merge
+ */
+ public abstract AbstractColumnTupleReader createColumnReader(IColumnProjectionInfo columnProjectionInfo);
+
+ @Override
+ public final ITreeIndexTupleWriter createTupleWriter() {
+ throw new UnsupportedOperationException("Operation is not supported for " + getClass().getName());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java
new file mode 100644
index 0000000..0c19ce7
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Columnar Tuple Writer:
+ * <p>
+ * The writer does not write directly to the page(s) buffer but write to internal temporary buffers (provided by
+ * {@link IColumnWriteMultiPageOp} until the header page (or page0) is full or the number of tuples equals to the
+ * {@link #getMaxNumberOfTuples()}
+ * Then, the columns are flushed to disk.
+ * <p>
+ * Contract:
+ * - Initially, the writer has to set multiPageOp by calling {@link #init(IColumnWriteMultiPageOp)}
+ * - For each write, the caller should check if adding a tuple does not exceed the {@link #getMaxNumberOfTuples()} or
+ * the on-disk page size (called stopping condition)
+ * - If the stopping condition is reached, then {@link #flush(ByteBuffer)} needed to be called
+ * <p>
+ * Hyracks visibility:
+ * - Columns are written as blobs (i.e., not interpretable by Hyracks)
+ * - Hyracks only aware of where each column at
+ */
+public abstract class AbstractColumnTupleWriter extends AbstractTupleWriterDisabledMethods {
+ /**
+ * Set the writer with {@link IColumnWriteMultiPageOp} to allocate columns for their writers
+ *
+ * @param multiPageOp multiPageOp
+ */
+ public abstract void init(IColumnWriteMultiPageOp multiPageOp) throws HyracksDataException;
+
+ /**
+ * @return The current number of columns
+ */
+ public abstract int getNumberOfColumns();
+
+ /**
+ * Currently, a column offset takes 4-byte (fixed). But in the future, we can reformat the offsets. For example,
+ * we can store index-offset pairs if we encounter a sparse columns (i.e., most columns are just nulls).
+ *
+ * @return the size needed to store columns' offsets
+ */
+ public final int getColumnOffsetsSize() {
+ return Integer.BYTES * getNumberOfColumns();
+ }
+
+ /**
+ * @return maximum number of tuples to be stored per page (i.e., page0)
+ */
+ public abstract int getMaxNumberOfTuples();
+
+ /**
+ * @return page0 occupied space
+ */
+ public abstract int getOccupiedSpace();
+
+ /**
+ * Writes the tuple into a temporary internal buffers
+ *
+ * @param tuple The tuple to be written
+ */
+ public abstract void writeTuple(ITupleReference tuple) throws HyracksDataException;
+
+ /**
+ * Flush all columns from the internal buffers to the page buffer
+ *
+ * @return the allocated space used to write tuples
+ */
+ public abstract int flush(ByteBuffer pageZero) throws HyracksDataException;
+
+ /**
+ * Close the current writer and release all allocated temporary buffers
+ */
+ public abstract void close();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractTupleWriterDisabledMethods.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractTupleWriterDisabledMethods.java
new file mode 100644
index 0000000..abc5ef0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractTupleWriterDisabledMethods.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+
+/**
+ * Disable all row write methods
+ */
+public abstract class AbstractTupleWriterDisabledMethods implements ITreeIndexTupleWriter {
+ protected static final String UNSUPPORTED_OPERATION_MSG = "Operation is not supported for columnar tuple reader";
+
+ /* ***********************************************
+ * Disable write-related operations
+ * ***********************************************
+ */
+
+ @Override
+ public final ITreeIndexTupleReference createTupleReference() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int writeTuple(ITupleReference tuple, ByteBuffer targetBuf, int targetOff) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int writeTupleFields(ITupleReference tuple, int startField, int numFields, byte[] targetBuf,
+ int targetOff) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int bytesRequired(ITupleReference tuple, int startField, int numFields) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int getCopySpaceRequired(ITupleReference tuple) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void setUpdated(boolean isUpdated) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnBufferProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnBufferProvider.java
new file mode 100644
index 0000000..d0b5e12
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnBufferProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import java.nio.ByteBuffer;
+import java.util.Queue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+/**
+ * A proxy to call {@link IBufferCache} operations. Each column should have its own buffer provider
+ */
+public interface IColumnBufferProvider {
+ /**
+ * Calling this method would pin all the pages of the requested columns from the buffer cache
+ *
+ * @param frame the frame for Page0
+ */
+ void reset(ColumnBTreeReadLeafFrame frame) throws HyracksDataException;
+
+ /**
+ * Return all the pages for a column
+ *
+ * @param buffers queue for all pages of a column
+ */
+ void readAll(Queue<ByteBuffer> buffers) throws HyracksDataException;
+
+ /**
+ * Release all the column pages (i.e., unpin all column pages)
+ */
+ void releaseAll() throws HyracksDataException;
+
+ /**
+ * @return a buffer of a column (in case there is only a single page for a column)
+ */
+ ByteBuffer getBuffer();
+
+ /**
+ * @return the actual length (in bytes) for all the column's pages
+ */
+ int getLength();
+
+ /**
+ * @return the column index
+ */
+ int getColumnIndex();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnManager.java
new file mode 100644
index 0000000..278ea03
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnManager.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+
+public interface IColumnManager {
+ /**
+ * Activate the columnar manager for an empty dataset
+ *
+ * @return empty column metadata
+ */
+ IColumnMetadata activate() throws HyracksDataException;
+
+ /**
+ * Activate the column manager for a non-empty dataset
+ *
+ * @param metadata column metadata value from the latest component metadata
+ * @return latest column metadata
+ */
+ IColumnMetadata activate(IValueReference metadata) throws HyracksDataException;
+
+ /**
+ * Create merge column metadata for a newly created merge component
+ *
+ * @param metadata latest column metadata value stored in the metadata page
+ * @param componentsTuples tuples of the merging components
+ * @return column metadata for a new merged component
+ */
+ IColumnMetadata createMergeColumnMetadata(IValueReference metadata, List<IColumnTupleIterator> componentsTuples)
+ throws HyracksDataException;
+
+ /**
+ * Create tuple projector for reading the merging components. The merge tuple projector will return all columns
+ *
+ * @return merge tuple projector
+ */
+ IColumnTupleProjector getMergeColumnProjector();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnManagerFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnManagerFactory.java
new file mode 100644
index 0000000..a2dfbcf
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnManagerFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
+
+public interface IColumnManagerFactory extends Serializable, IJsonSerializable {
+ /**
+ * @return a new instance of {@link IColumnManager}
+ */
+ IColumnManager createColumnManager();
+
+ /**
+ * Get column tuple reader/writer for the {@link LSMIOOperationType#LOAD}
+ */
+ AbstractColumnTupleReaderWriterFactory getLoadColumnTupleReaderWriterFactory();
+
+ /**
+ * Get column tuple reader/writer for the {@link LSMIOOperationType#FLUSH}
+ */
+ AbstractColumnTupleReaderWriterFactory getFlushColumnTupleReaderWriterFactory();
+
+ /**
+ * Get column tuple reader/writer for the {@link LSMIOOperationType#MERGE}
+ */
+ AbstractColumnTupleReaderWriterFactory createMergeColumnTupleReaderWriterFactory();
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnMetadata.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnMetadata.java
new file mode 100644
index 0000000..4c23b97
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnMetadata.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.util.annotations.NotThreadSafe;
+
+/**
+ * A holder for the columnar metadata.
+ * Modifications on the columnar metadata are not thread safe.
+ */
+@NotThreadSafe
+public interface IColumnMetadata {
+ /**
+ * @return a serialized version of the columns metadata
+ */
+ IValueReference serializeColumnsMetadata() throws HyracksDataException;
+
+ /**
+ * abort in case of an error. This should clean up any artifact
+ */
+ void abort() throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnReadMultiPageOp.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnReadMultiPageOp.java
new file mode 100644
index 0000000..f43a6e9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnReadMultiPageOp.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+/**
+ * A proxy to call {@link IBufferCache} read columns' pages
+ * Implementer should be aware to unpin all pages in case of an error
+ */
+public interface IColumnReadMultiPageOp {
+ /**
+ * Pin a column page
+ *
+ * @return a page that belongs to a column
+ */
+ ICachedPage pin(int pageId) throws HyracksDataException;
+
+ /**
+ * Unpin a pinned column page
+ */
+ void unpin(ICachedPage page) throws HyracksDataException;
+
+ /**
+ * Return {@link IBufferCache} page size
+ *
+ * @see IBufferCache#getPageSize()
+ */
+ int getPageSize();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnTupleIterator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnTupleIterator.java
new file mode 100644
index 0000000..0c95500
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnTupleIterator.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.ILSMIndexCursor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+/**
+ * A tuple representation that combines all the columns. It simply provides a way to iterate over tuples for a given
+ * set that could span multiple pages.
+ */
+public interface IColumnTupleIterator extends ILSMTreeTupleReference, Comparable<IColumnTupleIterator> {
+ /**
+ * Reset the iterator starting at the provided index
+ *
+ * @param startIndex start from the tuple at this index
+ */
+ void reset(int startIndex) throws HyracksDataException;
+
+ /**
+ * Mark {@link IColumnTupleIterator} as consumed
+ */
+ void consume();
+
+ /**
+ * @return true if the {@link IColumnTupleIterator} is consumed, false otherwise
+ */
+ boolean isConsumed();
+
+ /**
+ * Skip a number of tuples
+ *
+ * @param count the number of tuples that needed to be skipped
+ */
+ void skip(int count) throws HyracksDataException;
+
+ /**
+ * Move to the next tuple
+ */
+ void next() throws HyracksDataException;
+
+ /**
+ * Notifies that the last tuple has been consumed
+ */
+ void lastTupleReached() throws HyracksDataException;
+
+ /**
+ * The component index is the same as the index of a component in an {@link ILSMIndexCursor}
+ *
+ * @return From which {@link ILSMComponent} this iterator is for
+ */
+ int getComponentIndex();
+
+ /**
+ * Calls {@link IBufferCache#unpin(ICachedPage)} for all columns' pages
+ */
+ void unpinColumnsPages() throws HyracksDataException;
+
+ void close();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnWriteMultiPageOp.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnWriteMultiPageOp.java
new file mode 100644
index 0000000..2309fe1
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnWriteMultiPageOp.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+/**
+ * A proxy to call {@link IBufferCache} writing methods
+ * <p>
+ * An instance of this interface is responsible for returning all confiscated pages back to {@link IBufferCache} upon
+ * failures. Temporary buffers should be returned to the {@link IBufferCache} once the multi-page operation is finished.
+ * <p>
+ * Users of an instance of this interface should not expect the temporary buffers will last after the multi-page
+ * operation is finished.
+ */
+public interface IColumnWriteMultiPageOp {
+ /**
+ * @return a buffer that correspond to a page in a file
+ */
+ ByteBuffer confiscatePersistent() throws HyracksDataException;
+
+ /**
+ * Persist all confiscated persistent buffers to disk
+ */
+ void persist() throws HyracksDataException;
+
+ /**
+ * @return the number confiscated persistent pages
+ */
+ int getNumberOfPersistentBuffers();
+
+ /**
+ * @return a {@link IBufferCache}-backed buffer for temporary use
+ */
+ ByteBuffer confiscateTemporary() throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnProjectionInfo.java
similarity index 62%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnProjectionInfo.java
index 3c1a24d..1506433 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnProjectionInfo.java
@@ -16,20 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.hyracks.storage.am.lsm.btree.column.api.projection;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Gets information about the requested columns
*/
-public interface IProjectionInfo<T> {
+public interface IColumnProjectionInfo {
/**
- * @return projected values' information
+ * @param ordinal position of the requested column
+ * @return column index given the ordinal number of the requested column
*/
- T getProjectionInfo();
+ int getColumnIndex(int ordinal);
/**
- * @return a copy of the {@link IProjectionInfo}
+ * @return total number of requested columns
*/
- IProjectionInfo<T> createCopy();
+ int getNumberOfProjectedColumns();
+
+ /**
+ * @return number of primary keys
+ */
+ int getNumberOfPrimaryKeys();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnTupleProjector.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnTupleProjector.java
new file mode 100644
index 0000000..c1301da
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnTupleProjector.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.api.projection;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+import org.apache.hyracks.storage.common.projection.ITupleProjector;
+
+/**
+ * A specialized {@link ITupleProjector} for columnar datasets
+ */
+public interface IColumnTupleProjector extends ITupleProjector {
+ /**
+ * Create projection information
+ *
+ * @param columnMetadata the latest component's raw column metadata as stored in {@link IComponentMetadata}
+ * @return projection information
+ */
+ IColumnProjectionInfo createProjectionInfo(IValueReference columnMetadata) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResource.java
new file mode 100644
index 0000000..56090bb
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResource.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.dataflow;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManagerFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.utils.LSMColumnBTreeUtil;
+import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResource;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
+import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class LSMColumnBTreeLocalResource extends LSMBTreeLocalResource {
+ private final IColumnManagerFactory columnManagerFactory;
+
+ public LSMColumnBTreeLocalResource(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+ int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, String path,
+ IStorageManager storageManager, ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, int[] btreeFields, ILSMOperationTrackerFactory opTrackerProvider,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory, IVirtualBufferCacheProvider vbcProvider,
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+ ICompressorDecompressorFactory compressorDecompressorFactory, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector, boolean isSecondaryNoIncrementalMaintenance,
+ IColumnManagerFactory columnManagerFactory) {
+ super(typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, true, path, storageManager,
+ mergePolicyFactory, mergePolicyProperties, null, null, btreeFields, null, opTrackerProvider,
+ ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory, vbcProvider,
+ ioSchedulerProvider, true, compressorDecompressorFactory, true, nullTypeTraits, nullIntrospector,
+ isSecondaryNoIncrementalMaintenance);
+ this.columnManagerFactory = columnManagerFactory;
+ }
+
+ private LSMColumnBTreeLocalResource(IPersistedResourceRegistry registry, JsonNode json, int[] bloomFilterKeyFields,
+ double bloomFilterFalsePositiveRate, boolean isPrimary, int[] btreeFields,
+ ICompressorDecompressorFactory compressorDecompressorFactory, boolean hasBloomFilter,
+ boolean isSecondaryNoIncrementalMaintenance, IColumnManagerFactory columnManagerFactory)
+ throws HyracksDataException {
+ super(registry, json, bloomFilterKeyFields, bloomFilterFalsePositiveRate, isPrimary, btreeFields,
+ compressorDecompressorFactory, hasBloomFilter, isSecondaryNoIncrementalMaintenance);
+ this.columnManagerFactory = columnManagerFactory;
+ }
+
+ @Override
+ public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
+ IIOManager ioManager = serviceCtx.getIoManager();
+ FileReference file = ioManager.resolve(path);
+ List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
+ ioOpCallbackFactory.initialize(serviceCtx, this);
+ pageWriteCallbackFactory.initialize(serviceCtx, this);
+ return LSMColumnBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx),
+ typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+ opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx),
+ ioOpCallbackFactory, pageWriteCallbackFactory, btreeFields, metadataPageManagerFactory, false,
+ serviceCtx.getTracer(), compressorDecompressorFactory, nullTypeTraits, nullIntrospector,
+ columnManagerFactory);
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ int[] bloomFilterKeyFields = OBJECT_MAPPER.convertValue(json.get("bloomFilterKeyFields"), int[].class);
+ double bloomFilterFalsePositiveRate = json.get("bloomFilterFalsePositiveRate").asDouble();
+ boolean isPrimary = json.get("isPrimary").asBoolean();
+ boolean hasBloomFilter = getOrDefaultHasBloomFilter(json, isPrimary);
+ int[] btreeFields = OBJECT_MAPPER.convertValue(json.get("btreeFields"), int[].class);
+ JsonNode compressorDecompressorNode = json.get("compressorDecompressorFactory");
+ ICompressorDecompressorFactory compDecompFactory = (ICompressorDecompressorFactory) registry
+ .deserializeOrDefault(compressorDecompressorNode, NoOpCompressorDecompressorFactory.class);
+ JsonNode columnManagerFactoryNode = json.get("columnManagerFactory");
+ boolean isSecondaryNoIncrementalMaintenance =
+ getOrDefaultBoolean(json, "isSecondaryNoIncrementalMaintenance", false);
+ IColumnManagerFactory columnManagerFactory =
+ (IColumnManagerFactory) registry.deserialize(columnManagerFactoryNode);
+ return new LSMColumnBTreeLocalResource(registry, json, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
+ isPrimary, btreeFields, compDecompFactory, hasBloomFilter, isSecondaryNoIncrementalMaintenance,
+ columnManagerFactory);
+ }
+
+ @Override
+ protected void appendToJson(final ObjectNode json, IPersistedResourceRegistry registry)
+ throws HyracksDataException {
+ super.appendToJson(json, registry);
+ json.putPOJO("columnManagerFactory", columnManagerFactory.toJson(registry));
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResourceFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResourceFactory.java
new file mode 100644
index 0000000..eccb7c2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResourceFactory.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.dataflow;
+
+import java.util.Map;
+
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManagerFactory;
+import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResourceFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
+import org.apache.hyracks.storage.am.lsm.common.dataflow.LsmResource;
+import org.apache.hyracks.storage.common.IStorageManager;
+
+public class LSMColumnBTreeLocalResourceFactory extends LSMBTreeLocalResourceFactory {
+ private static final long serialVersionUID = -676367767925618165L;
+ private final IColumnManagerFactory columnManagerFactory;
+
+ public LSMColumnBTreeLocalResourceFactory(IStorageManager storageManager, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory, IVirtualBufferCacheProvider vbcProvider,
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
+ int[] btreeFields, ICompressorDecompressorFactory compressorDecompressorFactory, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector, boolean isSecondaryNoIncrementalMaintenance,
+ IColumnManagerFactory columnManagerFactory) {
+ super(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields,
+ opTrackerFactory, ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory,
+ vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, true, bloomFilterKeyFields,
+ bloomFilterFalsePositiveRate, true, btreeFields, compressorDecompressorFactory, true, nullTypeTraits,
+ nullIntrospector, isSecondaryNoIncrementalMaintenance);
+ this.columnManagerFactory = columnManagerFactory;
+ }
+
+ @Override
+ public LsmResource createResource(FileReference fileRef) {
+ return new LSMColumnBTreeLocalResource(typeTraits, cmpFactories, bloomFilterKeyFields,
+ bloomFilterFalsePositiveRate, fileRef.getRelativePath(), storageManager, mergePolicyFactory,
+ mergePolicyProperties, btreeFields, opTrackerProvider, ioOpCallbackFactory, pageWriteCallbackFactory,
+ metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, compressorDecompressorFactory,
+ nullTypeTraits, nullIntrospector, isSecondaryNoIncrementalMaintenance, columnManagerFactory);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
new file mode 100644
index 0000000..9aeafa4
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ISlotManager;
+import org.apache.hyracks.storage.am.common.api.ISplitKey;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
+
+/**
+ * Disable all unsupported/unused operations
+ */
+public abstract class AbstractColumnBTreeLeafFrame implements ITreeIndexFrame {
+ private static final String UNSUPPORTED_OPERATION_MSG = "Operation is not supported";
+
+ /*
+ * Remap the BTreeNSMFrame pointers for columnar pages
+ */
+ //Same as before
+ public static final int TUPLE_COUNT_OFFSET = Constants.TUPLE_COUNT_OFFSET;
+ //Previously Renaming
+ public static final int NUMBER_OF_COLUMNS_OFFSET = Constants.FREE_SPACE_OFFSET;
+ //Previously first four byte of LSN.
+ public static final int LEFT_MOST_KEY_OFFSET = Constants.RESERVED_HEADER_SIZE;
+ //Previously last four byte of LSN.
+ public static final int RIGHT_MOST_KEY_OFFSET = LEFT_MOST_KEY_OFFSET + 4;
+ /**
+ * Currently, a column offset takes 4-byte (fixed). But in the future, we can reformat the offsets. For example,
+ * we can store index-offset pairs if we encounter a sparse columns (i.e., most columns are just nulls). This
+ * reformatting could be indicated by the FLAG byte.
+ *
+ * @see AbstractColumnTupleWriter#getColumnOffsetsSize()
+ */
+ public static final int SIZE_OF_COLUMNS_OFFSETS_OFFSET = RIGHT_MOST_KEY_OFFSET + 4;
+ //Total number of columns pages
+ public static final int NUMBER_OF_COLUMN_PAGES = SIZE_OF_COLUMNS_OFFSETS_OFFSET + 4;
+ //A flag (used in NSM to indicate small and large pages). We can reuse it as explained above
+ public static final int FLAG_OFFSET = NUMBER_OF_COLUMN_PAGES + 4;
+ public static final int NEXT_LEAF_OFFSET = FLAG_OFFSET + 1;
+ public static final int HEADER_SIZE = NEXT_LEAF_OFFSET + 4;
+
+ protected final ITreeIndexTupleWriter rowTupleWriter;
+
+ protected MultiComparator cmp;
+ protected ICachedPage page;
+ protected ByteBuffer buf;
+
+ AbstractColumnBTreeLeafFrame(ITreeIndexTupleWriter rowTupleWriter) {
+ this.rowTupleWriter = rowTupleWriter;
+ }
+
+ /* ****************************************************************************
+ * Needed by both read and write
+ * ****************************************************************************
+ */
+
+ @Override
+ public final ITreeIndexTupleWriter getTupleWriter() {
+ return rowTupleWriter;
+ }
+
+ @Override
+ public final void setMultiComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public final void setPage(ICachedPage page) {
+ this.page = page;
+ this.buf = page.getBuffer();
+ buf.clear();
+ buf.position(HEADER_SIZE);
+ }
+
+ @Override
+ public final ICachedPage getPage() {
+ return page;
+ }
+
+ @Override
+ public final ByteBuffer getBuffer() {
+ return buf;
+ }
+
+ @Override
+ public final boolean isLeaf() {
+ return true;
+ }
+
+ @Override
+ public final boolean isInterior() {
+ return false;
+ }
+
+ @Override
+ public final int getPageHeaderSize() {
+ return HEADER_SIZE;
+ }
+
+ /* ****************************************************************************
+ * Operations that are needed by either read or write
+ * ****************************************************************************
+ */
+
+ @Override
+ public void initBuffer(byte level) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public int getTupleCount() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public ITupleReference getLeftmostTuple() throws HyracksDataException {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public ITupleReference getRightmostTuple() throws HyracksDataException {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ /* ****************************************************************************
+ * Unsupported Operations
+ * ****************************************************************************
+ */
+
+ @Override
+ public final String printHeader() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final byte getLevel() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void setLevel(byte level) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int getBytesRequiredToWriteTuple(ITupleReference tuple) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) throws HyracksDataException {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void insert(ITupleReference tuple, int tupleIndex) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final FrameOpSpaceStatus hasSpaceUpdate(ITupleReference newTuple, int oldTupleIndex) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void update(ITupleReference newTuple, int oldTupleIndex, boolean inPlace) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void delete(ITupleReference tuple, int tupleIndex) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final boolean compact() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final boolean compress() throws HyracksDataException {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int getTupleOffset(int slotNum) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int getTotalFreeSpace() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void setPageLsn(long pageLsn) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final long getPageLsn() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int getMaxTupleSize(int pageSize) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final ISlotManager getSlotManager() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final int getSlotSize() {
+ return 0;
+ }
+
+ @Override
+ public final int getFreeSpaceOff() {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void setFreeSpaceOff(int freeSpace) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
new file mode 100644
index 0000000..fcee22c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.storage.am.btree.impls.DiskBTree;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.NoOpIndexCursorStats;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+
+public class ColumnBTree extends DiskBTree {
+ public ColumnBTree(IBufferCache bufferCache, IPageManager freePageManager,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file) {
+ super(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
+ }
+
+ @Override
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+ boolean checkIfEmptyIndex, IPageWriteCallback callback) {
+ throw new IllegalAccessError("Missing write column metadata");
+ }
+
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, IPageWriteCallback callback,
+ IColumnMetadata columnMetadata) throws HyracksDataException {
+ ColumnBTreeLeafFrameFactory columnLeafFrameFactory = (ColumnBTreeLeafFrameFactory) leafFrameFactory;
+ ColumnBTreeWriteLeafFrame writeLeafFrame = columnLeafFrameFactory.createWriterFrame(columnMetadata);
+ return new ColumnBTreeBulkloader(fillFactor, verifyInput, callback, this, writeLeafFrame);
+ }
+
+ @Override
+ public BTreeAccessor createAccessor(IIndexAccessParameters iap) {
+ throw new IllegalArgumentException("Use createAccessor(IIndexAccessParameters, int, IColumnTupleProjector)");
+ }
+
+ public BTreeAccessor createAccessor(IIndexAccessParameters iap, int index, IColumnProjectionInfo projectionInfo) {
+ return new ColumnBTreeAccessor(this, iap, index, projectionInfo);
+ }
+
+ public class ColumnBTreeAccessor extends DiskBTreeAccessor {
+ private final int index;
+ private final IColumnProjectionInfo projectionInfo;
+
+ public ColumnBTreeAccessor(ColumnBTree btree, IIndexAccessParameters iap, int index,
+ IColumnProjectionInfo projectionInfo) {
+ super(btree, iap);
+ this.index = index;
+ this.projectionInfo = projectionInfo;
+ }
+
+ @Override
+ public ITreeIndexCursor createSearchCursor(boolean exclusive) {
+ ColumnBTreeLeafFrameFactory columnLeafFrameFactory = (ColumnBTreeLeafFrameFactory) leafFrameFactory;
+ ColumnBTreeReadLeafFrame readLeafFrame = columnLeafFrameFactory.createReadFrame(projectionInfo);
+ return new ColumnBTreeRangeSearchCursor(readLeafFrame, (IIndexCursorStats) iap.getParameters()
+ .getOrDefault(HyracksConstants.INDEX_CURSOR_STATS, NoOpIndexCursorStats.INSTANCE), index);
+ }
+
+ @Override
+ public ITreeIndexCursor createPointCursor(boolean exclusive, boolean stateful) {
+ ColumnBTreeLeafFrameFactory columnLeafFrameFactory = (ColumnBTreeLeafFrameFactory) leafFrameFactory;
+ ColumnBTreeReadLeafFrame readLeafFrame = columnLeafFrameFactory.createReadFrame(projectionInfo);
+ return new ColumnBTreePointSearchCursor(readLeafFrame, (IIndexCursorStats) iap.getParameters()
+ .getOrDefault(HyracksConstants.INDEX_CURSOR_STATS, NoOpIndexCursorStats.INSTANCE), index);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java
new file mode 100644
index 0000000..48bd180
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.btree.impls.BTreeNSMBulkLoader;
+import org.apache.hyracks.storage.am.btree.impls.BTreeSplitKey;
+import org.apache.hyracks.storage.am.common.api.ISplitKey;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.impls.NodeFrontier;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.common.buffercache.CachedPage;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements IColumnWriteMultiPageOp {
+ private final List<CachedPage> columnsPages;
+ private final List<CachedPage> tempConfiscatedPages;
+ private final ColumnBTreeWriteLeafFrame columnarFrame;
+ private final AbstractColumnTupleWriter columnWriter;
+ private final ISplitKey lowKey;
+ private boolean setLowKey;
+ private int tupleCount;
+
+ public ColumnBTreeBulkloader(float fillFactor, boolean verifyInput, IPageWriteCallback callback, ITreeIndex index,
+ ITreeIndexFrame leafFrame) throws HyracksDataException {
+ super(fillFactor, verifyInput, callback, index, leafFrame);
+ columnsPages = new ArrayList<>();
+ tempConfiscatedPages = new ArrayList<>();
+ columnarFrame = (ColumnBTreeWriteLeafFrame) leafFrame;
+ columnWriter = columnarFrame.getColumnTupleWriter();
+ columnWriter.init(this);
+ lowKey = new BTreeSplitKey(tupleWriter.createTupleReference());
+ lowKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
+ setLowKey = true;
+ }
+
+ @Override
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ if (isFull(tuple)) {
+ writeFullLeafPage();
+ confiscateNewLeafPage();
+ }
+ //Save the key of the last inserted tuple
+ setMinMaxKeys(tuple);
+ columnWriter.writeTuple(tuple);
+ tupleCount++;
+ }
+
+ @Override
+ protected ITreeIndexTupleReference createTupleReference() {
+ return tupleWriter.createTupleReference();
+ }
+
+ private boolean isFull(ITupleReference tuple) {
+ if (tupleCount == 0) {
+ return false;
+ } else if (tupleCount >= columnWriter.getMaxNumberOfTuples()) {
+ //We reached the maximum number of tuples
+ return true;
+ }
+ int requiredFreeSpace = AbstractColumnBTreeLeafFrame.HEADER_SIZE;
+ //Columns' Offsets
+ requiredFreeSpace += columnWriter.getColumnOffsetsSize();
+ //Occupied space from previous writes
+ requiredFreeSpace += columnWriter.getOccupiedSpace();
+ //min and max tuples' sizes
+ requiredFreeSpace += lowKey.getTuple().getTupleSize() + splitKey.getTuple().getTupleSize();
+ //New tuple required space
+ requiredFreeSpace += columnWriter.bytesRequired(tuple);
+ return bufferCache.getPageSize() <= requiredFreeSpace;
+ }
+
+ private void setMinMaxKeys(ITupleReference tuple) {
+ //Set max key
+ setSplitKey(splitKey, tuple);
+ if (setLowKey) {
+ setSplitKey(lowKey, tuple);
+ lowKey.getTuple().resetByTupleOffset(lowKey.getBuffer().array(), 0);
+ setLowKey = false;
+ }
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ if (tupleCount > 0) {
+ splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0);
+ columnarFrame.flush(columnWriter, tupleCount, this, lowKey.getTuple(), splitKey.getTuple());
+ }
+ columnWriter.close();
+ //We are done, return any temporary confiscated pages
+ for (ICachedPage page : tempConfiscatedPages) {
+ bufferCache.returnPage(page, false);
+ }
+ tempConfiscatedPages.clear();
+ //Where Page0 and columns pages will be written
+ super.end();
+ }
+
+ @Override
+ protected void writeFullLeafPage() throws HyracksDataException {
+ NodeFrontier leafFrontier = nodeFrontiers.get(0);
+ splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0);
+ splitKey.setLeftPage(leafFrontier.pageId);
+ if (tupleCount > 0) {
+ //We need to flush columns to confiscate all columns pages first before calling propagateBulk
+ columnarFrame.flush(columnWriter, tupleCount, this, lowKey.getTuple(), splitKey.getTuple());
+ }
+
+ propagateBulk(1, pagesToWrite);
+
+ //Take a page for the next leaf
+ leafFrontier.pageId = freePageManager.takePage(metaFrame);
+ columnarFrame.setNextLeaf(leafFrontier.pageId);
+
+ /*
+ * Write columns' pages first to ensure they (columns' pages) are written before pageZero.
+ * It ensures pageZero does not land in between columns' pages if compression is enabled
+ */
+ writeColumnsPages();
+ //Then write page0
+ write(leafFrontier.page);
+
+ //Write interior nodes after writing columns pages
+ for (ICachedPage c : pagesToWrite) {
+ write(c);
+ }
+
+ pagesToWrite.clear();
+ splitKey.setRightPage(leafFrontier.pageId);
+ setLowKey = true;
+ tupleCount = 0;
+ }
+
+ @Override
+ protected void writeLastLeaf(ICachedPage page) throws HyracksDataException {
+ /*
+ * Write columns' pages first to ensure they (columns' pages) are written before pageZero.
+ * It ensures pageZero does not land in between columns' pages if compression is enabled
+ */
+ writeColumnsPages();
+ super.writeLastLeaf(page);
+ }
+
+ private void writeColumnsPages() throws HyracksDataException {
+ for (ICachedPage c : columnsPages) {
+ write(c);
+ }
+ columnsPages.clear();
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ for (ICachedPage page : columnsPages) {
+ bufferCache.returnPage(page, false);
+ }
+
+ for (ICachedPage page : tempConfiscatedPages) {
+ bufferCache.returnPage(page, false);
+ }
+ super.abort();
+ }
+
+ private void setSplitKey(ISplitKey splitKey, ITupleReference tuple) {
+ int splitKeySize = tupleWriter.bytesRequired(tuple, 0, cmp.getKeyFieldCount());
+ splitKey.initData(splitKeySize);
+ tupleWriter.writeTupleFields(tuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer().array(), 0);
+ }
+
+ /*
+ * ***********************************************************
+ * IColumnWriteMultiPageOp
+ * ***********************************************************
+ */
+
+ @Override
+ public ByteBuffer confiscatePersistent() throws HyracksDataException {
+ int pageId = freePageManager.takePage(metaFrame);
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, pageId);
+ CachedPage page = (CachedPage) bufferCache.confiscatePage(dpid);
+ columnsPages.add(page);
+ return page.getBuffer();
+ }
+
+ @Override
+ public void persist() throws HyracksDataException {
+ writeColumnsPages();
+ }
+
+ @Override
+ public int getNumberOfPersistentBuffers() {
+ return columnsPages.size();
+ }
+
+ @Override
+ public ByteBuffer confiscateTemporary() throws HyracksDataException {
+ CachedPage page = (CachedPage) bufferCache.confiscatePage(IBufferCache.INVALID_DPID);
+ tempConfiscatedPages.add(page);
+ return page.getBuffer();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeFactory.java
new file mode 100644
index 0000000..1b9e198
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+public class ColumnBTreeFactory extends TreeIndexFactory<ColumnBTree> {
+ public ColumnBTreeFactory(IIOManager ioManager, IBufferCache bufferCache,
+ IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ super(ioManager, bufferCache, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ fieldCount);
+ }
+
+ @Override
+ public ColumnBTree createIndexInstance(FileReference file) throws HyracksDataException {
+ return new ColumnBTree(bufferCache, freePageManagerFactory.createPageManager(bufferCache), interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount, file);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeLeafFrameFactory.java
new file mode 100644
index 0000000..31d85bd
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeLeafFrameFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReaderWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+
+public class ColumnBTreeLeafFrameFactory implements ITreeIndexFrameFactory {
+ private static final long serialVersionUID = 4136035898137820322L;
+ private final ITreeIndexTupleWriterFactory rowTupleWriterFactory;
+ private final AbstractColumnTupleReaderWriterFactory columnTupleWriterFactory;
+
+ public ColumnBTreeLeafFrameFactory(ITreeIndexTupleWriterFactory rowTupleWriterFactory,
+ AbstractColumnTupleReaderWriterFactory columnTupleWriterFactory) {
+ this.rowTupleWriterFactory = rowTupleWriterFactory;
+ this.columnTupleWriterFactory = columnTupleWriterFactory;
+ }
+
+ @Override
+ public ITreeIndexFrame createFrame() {
+ //Create a dummy leaf frame
+ return new BTreeNSMLeafFrame(rowTupleWriterFactory.createTupleWriter());
+ }
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return rowTupleWriterFactory;
+ }
+
+ public ColumnBTreeWriteLeafFrame createWriterFrame(IColumnMetadata columnMetadata) {
+ ITreeIndexTupleWriter rowTupleWriter = rowTupleWriterFactory.createTupleWriter();
+ AbstractColumnTupleWriter columnTupleWriter = columnTupleWriterFactory.createColumnWriter(columnMetadata);
+ return new ColumnBTreeWriteLeafFrame(rowTupleWriter, columnTupleWriter);
+ }
+
+ public ColumnBTreeReadLeafFrame createReadFrame(IColumnProjectionInfo columnProjectionInfo) {
+ ITreeIndexTupleWriter rowTupleWriter = rowTupleWriterFactory.createTupleWriter();
+ AbstractColumnTupleReader columnTupleReader = columnTupleWriterFactory.createColumnReader(columnProjectionInfo);
+ return new ColumnBTreeReadLeafFrame(rowTupleWriter, columnTupleReader);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreePointSearchCursor.java
new file mode 100644
index 0000000..c93e77e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreePointSearchCursor.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.api.IDiskBTreeStatefulPointSearchCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+public class ColumnBTreePointSearchCursor extends ColumnBTreeRangeSearchCursor
+ implements IDiskBTreeStatefulPointSearchCursor {
+
+ public ColumnBTreePointSearchCursor(ColumnBTreeReadLeafFrame frame, IIndexCursorStats stats, int index) {
+ super(frame, stats, index);
+ }
+
+ @Override
+ public void doClose() throws HyracksDataException {
+ pageId = IBufferCache.INVALID_PAGEID;
+ super.doClose();
+ }
+
+ @Override
+ public int getLastPageId() {
+ return pageId;
+ }
+
+ @Override
+ public void setCursorToNextKey(ISearchPredicate searchPred) throws HyracksDataException {
+ initCursorPosition(searchPred);
+ }
+
+ @Override
+ public ITreeIndexFrame getFrame() {
+ return frame;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeRangeSearchCursor.java
new file mode 100644
index 0000000..fe980cc
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeRangeSearchCursor.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.btree.impls.BTreeCursorInitialState;
+import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+public class ColumnBTreeRangeSearchCursor extends EnforcedIndexCursor
+ implements ITreeIndexCursor, IColumnReadMultiPageOp {
+
+ protected final ColumnBTreeReadLeafFrame frame;
+ protected final IColumnTupleIterator frameTuple;
+
+ protected IBufferCache bufferCache = null;
+ protected int fileId;
+
+ protected int pageId;
+ protected ICachedPage page0 = null;
+
+ protected final RangePredicate reusablePredicate;
+ protected MultiComparator originalKeyCmp;
+
+ protected RangePredicate pred;
+ protected ITupleReference lowKey;
+ protected ITupleReference highKey;
+ protected boolean firstNextCall;
+
+ protected final IIndexCursorStats stats;
+
+ public ColumnBTreeRangeSearchCursor(ColumnBTreeReadLeafFrame frame, IIndexCursorStats stats, int index) {
+ this.frame = frame;
+ this.frameTuple = frame.createTupleReference(index, this);
+ this.reusablePredicate = new RangePredicate();
+ this.stats = stats;
+ fileId = -1;
+ pageId = IBufferCache.INVALID_PAGEID;
+ }
+
+ @Override
+ public void doDestroy() throws HyracksDataException {
+ // No Op all resources are released in the close call
+ }
+
+ @Override
+ public ITupleReference doGetTuple() {
+ return frameTuple;
+ }
+
+ private void fetchNextLeafPage(int leafPage) throws HyracksDataException {
+ int nextLeafPage = leafPage;
+ do {
+ ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
+ stats.getPageCounter().update(1);
+ bufferCache.unpin(page0);
+ page0 = nextLeaf;
+ frame.setPage(page0);
+ frameTuple.reset(0);
+ nextLeafPage = frame.getNextLeaf();
+ } while (frame.getTupleCount() == 0 && nextLeafPage > 0);
+ }
+
+ @Override
+ public boolean doHasNext() throws HyracksDataException {
+ int nextLeafPage;
+ if (frameTuple.isConsumed() && !firstNextCall) {
+ frameTuple.lastTupleReached();
+ nextLeafPage = frame.getNextLeaf();
+ if (nextLeafPage >= 0) {
+ fetchNextLeafPage(nextLeafPage);
+ } else {
+ return false;
+ }
+ }
+ return isNextIncluded();
+ }
+
+ @Override
+ public void doNext() throws HyracksDataException {
+ //NoOp
+ }
+
+ @Override
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ // in case open is called multiple times without closing
+ if (page0 != null) {
+ releasePages();
+ }
+ originalKeyCmp = initialState.getOriginalKeyComparator();
+ page0 = initialState.getPage();
+ pageId = ((BTreeCursorInitialState) initialState).getPageId();
+ frame.setPage(page0);
+ frame.setMultiComparator(originalKeyCmp);
+ frameTuple.reset(0);
+ initCursorPosition(searchPred);
+ }
+
+ protected void initCursorPosition(ISearchPredicate searchPred) throws HyracksDataException {
+ pred = (RangePredicate) searchPred;
+ lowKey = pred.getLowKey();
+ highKey = pred.getHighKey();
+
+ reusablePredicate.setLowKeyComparator(originalKeyCmp);
+ reusablePredicate.setHighKeyComparator(pred.getHighKeyComparator());
+ reusablePredicate.setHighKey(pred.getHighKey(), pred.isHighKeyInclusive());
+ firstNextCall = true;
+ advanceTupleToLowKey();
+ }
+
+ protected boolean isNextIncluded() throws HyracksDataException {
+ if (firstNextCall) {
+ //The first call of frameTuple.next() was done during the opening of the cursor
+ firstNextCall = false;
+ return true;
+ } else if (frameTuple.isConsumed()) {
+ //All tuple were consumed
+ return false;
+ }
+ //Next tuple
+ frameTuple.next();
+ //Check whether the frameTuple is not consumed and also include the search key
+ return highKey == null || isLessOrEqual(frameTuple, highKey, pred.isHighKeyInclusive());
+ }
+
+ protected void advanceTupleToLowKey() throws HyracksDataException {
+ if (highKey != null && isLessOrEqual(highKey, frame.getLeftmostTuple(), !pred.isHighKeyInclusive())) {
+ /*
+ * Lowest key from the frame is greater than the requested highKey. No tuple will satisfy the search
+ * key. Consume the frameTuple to stop the search
+ */
+ firstNextCall = false;
+ frameTuple.consume();
+ return;
+ } else if (lowKey == null) {
+ //No range was specified.
+ frameTuple.next();
+ return;
+ }
+
+ //The lowKey is somewhere within the frame tuples
+ boolean stop = false;
+ int counter = 0;
+ while (!stop && !frameTuple.isConsumed()) {
+ frameTuple.next();
+ stop = isLessOrEqual(lowKey, frameTuple, pred.isLowKeyInclusive());
+ counter++;
+ }
+ //Advance all columns to the proper position
+ frameTuple.skip(counter - 1);
+ }
+
+ protected void releasePages() throws HyracksDataException {
+ //Unpin all column pages first
+ frameTuple.unpinColumnsPages();
+ if (page0 != null) {
+ bufferCache.unpin(page0);
+ }
+ }
+
+ private boolean isLessOrEqual(ITupleReference left, ITupleReference right, boolean inclusive)
+ throws HyracksDataException {
+ int cmp = originalKeyCmp.compare(left, right);
+ return cmp < 0 || cmp == 0 && inclusive;
+ }
+
+ @Override
+ public void doClose() throws HyracksDataException {
+ frameTuple.close();
+ releasePages();
+ page0 = null;
+ pred = null;
+ }
+
+ @Override
+ public void setBufferCache(IBufferCache bufferCache) {
+ this.bufferCache = bufferCache;
+ }
+
+ @Override
+ public void setFileId(int fileId) {
+ this.fileId = fileId;
+ }
+
+ @Override
+ public boolean isExclusiveLatchNodes() {
+ return false;
+ }
+
+ /*
+ * ***********************************************************
+ * IColumnReadMultiPageOp
+ * ***********************************************************
+ */
+ @Override
+ public ICachedPage pin(int pageId) throws HyracksDataException {
+ stats.getPageCounter().update(1);
+ return bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ }
+
+ @Override
+ public void unpin(ICachedPage page) throws HyracksDataException {
+ bufferCache.unpin(page);
+ }
+
+ @Override
+ public int getPageSize() {
+ return bufferCache.getPageSize();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
new file mode 100644
index 0000000..8872613
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.common.buffercache.CachedPage;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+public final class ColumnBTreeReadLeafFrame extends AbstractColumnBTreeLeafFrame {
+ private final AbstractColumnTupleReader columnarTupleReader;
+ private final ITreeIndexTupleReference leftMostTuple;
+ private final ITreeIndexTupleReference rightMostTuple;
+
+ public ColumnBTreeReadLeafFrame(ITreeIndexTupleWriter rowTupleWriter,
+ AbstractColumnTupleReader columnarTupleReader) {
+ super(rowTupleWriter);
+ this.columnarTupleReader = columnarTupleReader;
+ leftMostTuple = rowTupleWriter.createTupleReference();
+ rightMostTuple = rowTupleWriter.createTupleReference();
+ }
+
+ @Override
+ public ITupleReference getLeftmostTuple() {
+ if (getTupleCount() == 0) {
+ return null;
+ }
+ leftMostTuple.setFieldCount(cmp.getKeyFieldCount());
+ leftMostTuple.resetByTupleOffset(buf.array(), buf.getInt(LEFT_MOST_KEY_OFFSET));
+ return leftMostTuple;
+ }
+
+ @Override
+ public ITupleReference getRightmostTuple() {
+ if (getTupleCount() == 0) {
+ return null;
+ }
+ rightMostTuple.setFieldCount(cmp.getKeyFieldCount());
+ rightMostTuple.resetByTupleOffset(buf.array(), buf.getInt(RIGHT_MOST_KEY_OFFSET));
+ return rightMostTuple;
+ }
+
+ public IColumnTupleIterator createTupleReference(int index, IColumnReadMultiPageOp multiPageOp) {
+ return columnarTupleReader.createTupleIterator(this, index, multiPageOp);
+ }
+
+ @Override
+ public int getTupleCount() {
+ return buf.getInt(Constants.TUPLE_COUNT_OFFSET);
+ }
+
+ public int getPageId() {
+ return BufferedFileHandle.getPageId(((CachedPage) page).getDiskPageId());
+ }
+
+ public int getNumberOfColumns() {
+ return buf.getInt(NUMBER_OF_COLUMNS_OFFSET);
+ }
+
+ public int getColumnOffset(int columnIndex) {
+ if (columnIndex >= getNumberOfColumns()) {
+ throw new IndexOutOfBoundsException(columnIndex + " >= " + getNumberOfColumns());
+ }
+ return columnarTupleReader.getColumnOffset(buf, columnIndex);
+ }
+
+ AbstractColumnTupleReader getColumnarTupleReader() {
+ return columnarTupleReader;
+ }
+
+ int getNextLeaf() {
+ return buf.getInt(NEXT_LEAF_OFFSET);
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ throw new IllegalArgumentException("Use createTupleReference(int)");
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java
new file mode 100644
index 0000000..275fb0e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+public class ColumnBTreeWriteLeafFrame extends AbstractColumnBTreeLeafFrame {
+ private final AbstractColumnTupleWriter columnTupleWriter;
+
+ public ColumnBTreeWriteLeafFrame(ITreeIndexTupleWriter rowTupleWriter,
+ AbstractColumnTupleWriter columnTupleWriter) {
+ super(rowTupleWriter);
+ this.columnTupleWriter = columnTupleWriter;
+ }
+
+ @Override
+ public void initBuffer(byte level) {
+ buf.putInt(TUPLE_COUNT_OFFSET, 0);
+ buf.put(Constants.LEVEL_OFFSET, level);
+ buf.putInt(NUMBER_OF_COLUMNS_OFFSET, 0);
+ buf.putInt(LEFT_MOST_KEY_OFFSET, -1);
+ buf.putInt(RIGHT_MOST_KEY_OFFSET, -1);
+ buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, 0);
+ buf.putInt(NUMBER_OF_COLUMN_PAGES, 0);
+ buf.put(FLAG_OFFSET, (byte) 0);
+ buf.putInt(NEXT_LEAF_OFFSET, -1);
+ }
+
+ void flush(AbstractColumnTupleWriter columnWriter, int numberOfTuples, IColumnWriteMultiPageOp multiPageOp,
+ ITupleReference minKey, ITupleReference maxKey) throws HyracksDataException {
+ //Prepare the space for writing the columns' information such as the primary keys
+ buf.position(HEADER_SIZE);
+ //Write the columns' information including the columns' offsets and the primary keys
+ columnWriter.flush(buf);
+
+ //Write min and max keys
+ int offset = buf.position();
+ buf.putInt(LEFT_MOST_KEY_OFFSET, offset);
+ offset += rowTupleWriter.writeTuple(minKey, buf.array(), offset);
+ buf.putInt(RIGHT_MOST_KEY_OFFSET, offset);
+ rowTupleWriter.writeTuple(maxKey, buf.array(), offset);
+
+ //Write page information
+ int numberOfColumns = columnWriter.getNumberOfColumns();
+ buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples);
+ buf.putInt(NUMBER_OF_COLUMNS_OFFSET, numberOfColumns);
+ buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, columnWriter.getColumnOffsetsSize());
+ buf.putInt(NUMBER_OF_COLUMN_PAGES, multiPageOp.getNumberOfPersistentBuffers());
+ }
+
+ public AbstractColumnTupleWriter getColumnTupleWriter() {
+ return columnTupleWriter;
+ }
+
+ void setNextLeaf(int pageId) {
+ buf.putInt(NEXT_LEAF_OFFSET, pageId);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
new file mode 100644
index 0000000..048d9de
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.storage.am.common.api.IExtendedModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManager;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+import org.apache.hyracks.storage.am.lsm.btree.column.utils.ColumnUtil;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeRangeSearchCursor;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
+
+public class LSMColumnBTree extends LSMBTree {
+ private static final ICursorFactory CURSOR_FACTORY = LSMColumnBTreeSearchCursor::new;
+ private final IColumnManager columnManager;
+ private final ILSMDiskComponentFactory mergeComponentFactory;
+ /**
+ * This column metadata only used during flush and dataset bulkload operations. We cannot have more than one
+ * thread to do a flush/dataset bulkload. Do not use it for search/scan. Instead, use the latest component
+ * metadata of the operational disk components.
+ *
+ * @see LSMColumnBTreeOpContext#createProjectionInfo()
+ */
+ private IColumnMetadata columnMetadata;
+
+ public LSMColumnBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache diskBufferCache,
+ ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
+ ILSMDiskComponentFactory mergeComponentFactory, ILSMDiskComponentFactory bulkloadComponentFactory,
+ double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ int[] btreeFields, ITracer tracer, IColumnManager columnManager) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
+ diskBufferCache, fileManager, componentFactory, bulkloadComponentFactory, null, null, null,
+ bloomFilterFalsePositiveRate, fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler,
+ ioOpCallbackFactory, pageWriteCallbackFactory, true, true, btreeFields, null, true, false, tracer);
+ this.columnManager = columnManager;
+ this.mergeComponentFactory = mergeComponentFactory;
+ }
+
+ @Override
+ public synchronized void activate() throws HyracksDataException {
+ super.activate();
+ if (diskComponents.isEmpty()) {
+ columnMetadata = columnManager.activate();
+ } else {
+ IComponentMetadata componentMetadata = diskComponents.get(0).getMetadata();
+ columnMetadata = columnManager.activate(ColumnUtil.getColumnMetadataCopy(componentMetadata));
+ }
+ }
+
+ @Override
+ public LSMColumnBTreeOpContext createOpContext(IIndexAccessParameters iap) {
+ int numBloomFilterKeyFields =
+ ((LSMColumnBTreeWithBloomFilterDiskComponentFactory) componentFactory).getBloomFilterKeyFields().length;
+ IColumnTupleProjector tupleProjector =
+ ColumnUtil.getTupleProjector(iap, columnManager.getMergeColumnProjector());
+ return new LSMColumnBTreeOpContext(this, memoryComponents, insertLeafFrameFactory, deleteLeafFrameFactory,
+ (IExtendedModificationOperationCallback) iap.getModificationCallback(),
+ iap.getSearchOperationCallback(), numBloomFilterKeyFields, getTreeFields(), getFilterFields(),
+ getHarness(), getFilterCmpFactories(), tracer, tupleProjector);
+ }
+
+ protected IColumnManager getColumnManager() {
+ return columnManager;
+ }
+
+ protected IColumnMetadata getColumnMetadata() {
+ return columnMetadata;
+ }
+
+ @Override
+ protected LSMBTreeRangeSearchCursor createCursor(AbstractLSMIndexOperationContext opCtx,
+ boolean returnDeletedTuples, IIndexCursorStats stats) {
+ return new LSMColumnBTreeRangeSearchCursor(opCtx, returnDeletedTuples, stats);
+ }
+
+ @Override
+ public LSMBTreeBatchPointSearchCursor createBatchPointSearchCursor(ILSMIndexOperationContext opCtx) {
+ return new LSMColumnBatchPointSearchCursor(opCtx);
+ }
+
+ @Override
+ protected ILSMDiskComponentFactory getMergeComponentFactory() {
+ return mergeComponentFactory;
+ }
+
+ @Override
+ public ICursorFactory getCursorFactory() {
+ return CURSOR_FACTORY;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeOpContext.java
new file mode 100644
index 0000000..8a33de1
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeOpContext.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.common.api.IExtendedModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples.ColumnAwareMultiComparator;
+import org.apache.hyracks.storage.am.lsm.btree.column.utils.ColumnUtil;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeOpContext;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.util.trace.ITracer;
+
+public class LSMColumnBTreeOpContext extends LSMBTreeOpContext {
+ private final IColumnTupleProjector projector;
+
+ public LSMColumnBTreeOpContext(ILSMIndex index, List<ILSMMemoryComponent> mutableComponents,
+ ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
+ IExtendedModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
+ int numBloomFilterKeyFields, int[] btreeFields, int[] filterFields, ILSMHarness lsmHarness,
+ IBinaryComparatorFactory[] filterCmpFactories, ITracer tracer, IColumnTupleProjector projector) {
+ super(index, mutableComponents, insertLeafFrameFactory, deleteLeafFrameFactory, modificationCallback,
+ searchCallback, numBloomFilterKeyFields, btreeFields, filterFields, lsmHarness, filterCmpFactories,
+ tracer);
+ this.projector = projector;
+ }
+
+ public IColumnProjectionInfo createProjectionInfo() throws HyracksDataException {
+ List<ILSMComponent> operationalComponents = getComponentHolder();
+ IComponentMetadata componentMetadata = null;
+ for (int i = 0; i < operationalComponents.size() && componentMetadata == null; i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ if (component.getType() == LSMComponentType.DISK) {
+ //Find the first on-disk component, which has the most recent column metadata.
+ componentMetadata = component.getMetadata();
+ }
+ }
+ if (componentMetadata != null) {
+ IValueReference columnMetadata = ColumnUtil.getColumnMetadataCopy(componentMetadata);
+ return projector.createProjectionInfo(columnMetadata);
+ }
+ //In-memory components only
+ return null;
+ }
+
+ @Override
+ protected MultiComparator createMultiComparator(IBinaryComparatorFactory[] cmpFactories) {
+ IBinaryComparator[] comparators = new IBinaryComparator[cmpFactories.length];
+ for (int i = 0; i < comparators.length; i++) {
+ comparators[i] = cmpFactories[i].createBinaryComparator();
+ }
+ return new ColumnAwareMultiComparator(comparators);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeRangeSearchCursor.java
new file mode 100644
index 0000000..75b2c72
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeRangeSearchCursor.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeRangeSearchCursor;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples.ColumnAwareDiskOnlyMultiComparator;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeRangeSearchCursor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.NoOpIndexCursorStats;
+
+public class LSMColumnBTreeRangeSearchCursor extends LSMBTreeRangeSearchCursor {
+ private final List<IColumnTupleIterator> componentTupleList;
+
+ public LSMColumnBTreeRangeSearchCursor(ILSMIndexOperationContext opCtx) {
+ this(opCtx, false, NoOpIndexCursorStats.INSTANCE);
+ }
+
+ public LSMColumnBTreeRangeSearchCursor(ILSMIndexOperationContext opCtx, boolean returnDeletedTuples,
+ IIndexCursorStats stats) {
+ super(opCtx, returnDeletedTuples, stats);
+ componentTupleList = new ArrayList<>();
+ }
+
+ @Override
+ protected BTreeAccessor createAccessor(LSMComponentType type, BTree btree, int index) throws HyracksDataException {
+ if (type == LSMComponentType.MEMORY) {
+ return super.createAccessor(type, btree, index);
+ }
+ ColumnBTree columnBTree = (ColumnBTree) btree;
+ LSMColumnBTreeOpContext columnOpCtx = (LSMColumnBTreeOpContext) opCtx;
+ return columnBTree.createAccessor(iap, index, columnOpCtx.createProjectionInfo());
+ }
+
+ @Override
+ protected IIndexCursor createCursor(LSMComponentType type, BTreeAccessor accessor) {
+ if (type == LSMComponentType.MEMORY) {
+ return super.createCursor(type, accessor);
+ }
+ ColumnBTreeRangeSearchCursor cursor = (ColumnBTreeRangeSearchCursor) accessor.createSearchCursor(false);
+ componentTupleList.add((IColumnTupleIterator) cursor.doGetTuple());
+ return cursor;
+ }
+
+ @Override
+ protected void markAsDeleted(PriorityQueueElement e) throws HyracksDataException {
+ if (isMemoryComponent[e.getCursorIndex()]) {
+ super.markAsDeleted(e);
+ return;
+ }
+ IColumnTupleIterator columnTuple = (IColumnTupleIterator) e.getTuple();
+ columnTuple.skip(1);
+ }
+
+ @Override
+ protected void setPriorityQueueComparator() {
+ if (!includeMutableComponent) {
+ cmp = new ColumnAwareDiskOnlyMultiComparator(cmp);
+ }
+ if (pqCmp == null || cmp != pqCmp.getMultiComparator()) {
+ pqCmp = new PriorityQueueComparator(cmp);
+ }
+ }
+
+ @Override
+ protected void excludeMemoryComponent() {
+ //Replace the comparator with disk only comparator
+ pqCmp.setMultiComparator(new ColumnAwareDiskOnlyMultiComparator(cmp));
+ }
+
+ @Override
+ protected int replaceFrom() {
+ //Disable replacing the in-memory component to disk component as the schema may change
+ //TODO at least allow the replacement when no schema changes occur
+ return -1;
+ }
+
+ /**
+ * @return we need the tuple references for vertical merges
+ */
+ public List<IColumnTupleIterator> getComponentTupleList() {
+ return componentTupleList;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeSearchCursor.java
new file mode 100644
index 0000000..6ca41d0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeSearchCursor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponentScanCursor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeSearchCursor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class LSMColumnBTreeSearchCursor extends LSMBTreeSearchCursor {
+ public LSMColumnBTreeSearchCursor(ILSMIndexOperationContext opCtx) {
+ super(new LSMColumnPointSearchCursor(opCtx), new LSMColumnBTreeRangeSearchCursor(opCtx),
+ new LSMBTreeDiskComponentScanCursor(opCtx));
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
new file mode 100644
index 0000000..57e162d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.column.utils.ColumnUtil;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeMergeOperation;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBloomFilterDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.ChainedLSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+
+public class LSMColumnBTreeWithBloomFilterDiskComponent extends LSMBTreeWithBloomFilterDiskComponent {
+
+ public LSMColumnBTreeWithBloomFilterDiskComponent(AbstractLSMIndex lsmIndex, BTree btree, BloomFilter bloomFilter,
+ ILSMComponentFilter filter) {
+ super(lsmIndex, btree, bloomFilter, filter);
+ }
+
+ @Override
+ public ChainedLSMDiskComponentBulkLoader createBulkLoader(ILSMIOOperation operation, float fillFactor,
+ boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
+ boolean cleanupEmptyComponent, IPageWriteCallback callback) throws HyracksDataException {
+ ChainedLSMDiskComponentBulkLoader chainedBulkLoader =
+ new ChainedLSMDiskComponentBulkLoader(operation, this, cleanupEmptyComponent);
+ if (withFilter && getLsmIndex().getFilterFields() != null) {
+ //Add filter writer if exists
+ chainedBulkLoader.addBulkLoader(createFilterBulkLoader());
+ }
+ //Add index bulkloader
+ chainedBulkLoader.addBulkLoader(createColumnIndexBulkLoader(operation, fillFactor, verifyInput, callback));
+
+ if (numElementsHint > 0) {
+ chainedBulkLoader.addBulkLoader(createBloomFilterBulkLoader(numElementsHint, callback));
+ }
+
+ callback.initialize(chainedBulkLoader);
+ return chainedBulkLoader;
+ }
+
+ private IChainedComponentBulkLoader createColumnIndexBulkLoader(ILSMIOOperation operation, float fillFactor,
+ boolean verifyInput, IPageWriteCallback callback) throws HyracksDataException {
+ LSMIOOperationType operationType = operation.getIOOpertionType();
+ LSMColumnBTree lsmColumnBTree = (LSMColumnBTree) getLsmIndex();
+ ColumnBTree columnBTree = (ColumnBTree) getIndex();
+ IColumnMetadata columnMetadata;
+ if (operationType == LSMIOOperationType.FLUSH || operationType == LSMIOOperationType.LOAD) {
+ columnMetadata = lsmColumnBTree.getColumnMetadata();
+ } else {
+ //Merge
+ LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
+ LSMColumnBTreeRangeSearchCursor cursor = (LSMColumnBTreeRangeSearchCursor) mergeOp.getCursor();
+ List<ILSMComponent> mergingComponents = mergeOp.getMergingComponents();
+ IComponentMetadata componentMetadata = mergingComponents.get(0).getMetadata();
+ IValueReference columnMetadataValue = ColumnUtil.getColumnMetadataCopy(componentMetadata);
+ columnMetadata = lsmColumnBTree.getColumnManager().createMergeColumnMetadata(columnMetadataValue,
+ cursor.getComponentTupleList());
+ }
+ IIndexBulkLoader bulkLoader = columnBTree.createBulkLoader(fillFactor, verifyInput, callback, columnMetadata);
+ return new LSMColumnIndexBulkloader(bulkLoader, columnMetadata, getMetadata());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponentFactory.java
new file mode 100644
index 0000000..2d86a73
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponentFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBloomFilterDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+
+/**
+ * We only create a BTree with a bloom filter
+ */
+public class LSMColumnBTreeWithBloomFilterDiskComponentFactory implements ILSMDiskComponentFactory {
+ private final TreeIndexFactory<ColumnBTree> btreeFactory;
+ private final BloomFilterFactory bloomFilterFactory;
+
+ public LSMColumnBTreeWithBloomFilterDiskComponentFactory(TreeIndexFactory<ColumnBTree> btreeFactory,
+ BloomFilterFactory bloomFilterFactory) {
+ this.btreeFactory = btreeFactory;
+ this.bloomFilterFactory = bloomFilterFactory;
+ }
+
+ @Override
+ public LSMBTreeWithBloomFilterDiskComponent createComponent(AbstractLSMIndex lsmIndex,
+ LSMComponentFileReferences cfr) throws HyracksDataException {
+ return new LSMColumnBTreeWithBloomFilterDiskComponent(lsmIndex,
+ btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
+ bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), null);
+ }
+
+ public int[] getBloomFilterKeyFields() {
+ return bloomFilterFactory.getBloomFilterKeyFields();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBatchPointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBatchPointSearchCursor.java
new file mode 100644
index 0000000..65b292b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBatchPointSearchCursor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class LSMColumnBatchPointSearchCursor extends LSMBTreeBatchPointSearchCursor {
+
+ public LSMColumnBatchPointSearchCursor(ILSMIndexOperationContext opCtx) {
+ super(opCtx);
+ }
+
+ @Override
+ protected BTreeAccessor createAccessor(LSMComponentType type, BTree btree, int index) throws HyracksDataException {
+ if (type == LSMComponentType.MEMORY) {
+ return super.createAccessor(type, btree, index);
+ }
+ ColumnBTree columnBTree = (ColumnBTree) btree;
+ LSMColumnBTreeOpContext columnOpCtx = (LSMColumnBTreeOpContext) opCtx;
+ return columnBTree.createAccessor(NoOpIndexAccessParameters.INSTANCE, index,
+ columnOpCtx.createProjectionInfo());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnIndexBulkloader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnIndexBulkloader.java
new file mode 100644
index 0000000..ba41227
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnIndexBulkloader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnMetadata;
+import org.apache.hyracks.storage.am.lsm.btree.column.utils.ColumnUtil;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexBulkLoader;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+
+public class LSMColumnIndexBulkloader extends LSMIndexBulkLoader {
+ private final IColumnMetadata columnMetadata;
+ private final IComponentMetadata componentMetadata;
+
+ public LSMColumnIndexBulkloader(IIndexBulkLoader bulkLoader, IColumnMetadata columnMetadata,
+ IComponentMetadata componentMetadata) {
+ super(bulkLoader);
+ this.columnMetadata = columnMetadata;
+ this.componentMetadata = componentMetadata;
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ ColumnUtil.putColumnsMetadataValue(columnMetadata.serializeColumnsMetadata(), componentMetadata);
+ super.end();
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ columnMetadata.abort();
+ super.abort();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnPointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnPointSearchCursor.java
new file mode 100644
index 0000000..e193232
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnPointSearchCursor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreePointSearchCursor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class LSMColumnPointSearchCursor extends LSMBTreePointSearchCursor {
+
+ public LSMColumnPointSearchCursor(ILSMIndexOperationContext opCtx) {
+ super(opCtx);
+ }
+
+ @Override
+ protected BTreeAccessor createAccessor(LSMComponentType type, BTree btree, int index) throws HyracksDataException {
+ if (type == LSMComponentType.MEMORY) {
+ return super.createAccessor(type, btree, index);
+ }
+ ColumnBTree columnBTree = (ColumnBTree) btree;
+ LSMColumnBTreeOpContext columnOpCtx = (LSMColumnBTreeOpContext) opCtx;
+ return columnBTree.createAccessor(NoOpIndexAccessParameters.INSTANCE, index,
+ columnOpCtx.createProjectionInfo());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
new file mode 100644
index 0000000..d0e1e1d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples;
+
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.HEADER_SIZE;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public abstract class AbstractColumnTupleReference implements IColumnTupleIterator {
+ private static final Logger LOGGER = LogManager.getLogger();
+ private static final String UNSUPPORTED_OPERATION_MSG = "Operation is not supported for column tuples";
+ private final int componentIndex;
+ private final ColumnBTreeReadLeafFrame frame;
+ private final IColumnBufferProvider[] primaryKeyBufferProviders;
+ private final IColumnBufferProvider[] buffersProviders;
+ private final int numberOfPrimaryKeys;
+ private int totalNumberOfMegaLeafNodes;
+ private int numOfSkippedMegaLeafNodes;
+ protected int tupleIndex;
+
+ /**
+ * Column tuple reference
+ *
+ * @param componentIndex LSM component index
+ * @param frame page0 frame
+ * @param info projection info
+ */
+ protected AbstractColumnTupleReference(int componentIndex, ColumnBTreeReadLeafFrame frame,
+ IColumnProjectionInfo info, IColumnReadMultiPageOp multiPageOp) {
+ this.componentIndex = componentIndex;
+ this.frame = frame;
+ numberOfPrimaryKeys = info.getNumberOfPrimaryKeys();
+
+ primaryKeyBufferProviders = new IColumnBufferProvider[numberOfPrimaryKeys];
+
+ for (int i = 0; i < numberOfPrimaryKeys; i++) {
+ primaryKeyBufferProviders[i] = new ColumnSingleBufferProvider(i);
+ }
+
+ int numberOfRequestedColumns = info.getNumberOfProjectedColumns();
+ buffersProviders = new IColumnBufferProvider[numberOfRequestedColumns];
+ for (int i = 0; i < numberOfRequestedColumns; i++) {
+ int columnIndex = info.getColumnIndex(i);
+ if (columnIndex >= numberOfPrimaryKeys) {
+ buffersProviders[i] = new ColumnMultiBufferProvider(columnIndex, multiPageOp);
+ } else {
+ buffersProviders[i] = new ColumnSingleBufferProvider(columnIndex);
+ }
+ }
+ totalNumberOfMegaLeafNodes = 0;
+ numOfSkippedMegaLeafNodes = 0;
+ }
+
+ @Override
+ public final void reset(int startIndex) throws HyracksDataException {
+ tupleIndex = startIndex;
+ ByteBuffer pageZero = frame.getBuffer();
+ pageZero.clear();
+ pageZero.position(HEADER_SIZE);
+
+ int numberOfTuples = frame.getTupleCount();
+ //Start new page and check whether we should skip reading non-key columns or not
+ boolean readColumnPages = startNewPage(pageZero, frame.getNumberOfColumns(), numberOfTuples);
+
+ //Start primary keys
+ for (int i = 0; i < numberOfPrimaryKeys; i++) {
+ IColumnBufferProvider provider = primaryKeyBufferProviders[i];
+ provider.reset(frame);
+ startPrimaryKey(provider, tupleIndex, i, numberOfTuples);
+ }
+
+ if (readColumnPages) {
+ for (int i = 0; i < buffersProviders.length; i++) {
+ IColumnBufferProvider provider = buffersProviders[i];
+ //Release previous pinned pages if any
+ provider.releaseAll();
+ provider.reset(frame);
+ startColumn(provider, tupleIndex, i, numberOfTuples);
+ }
+ } else {
+ numOfSkippedMegaLeafNodes++;
+ }
+ totalNumberOfMegaLeafNodes++;
+ }
+
+ protected abstract boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples);
+
+ protected abstract void startPrimaryKey(IColumnBufferProvider bufferProvider, int startIndex, int ordinal,
+ int numberOfTuples) throws HyracksDataException;
+
+ protected abstract void startColumn(IColumnBufferProvider buffersProvider, int startIndex, int ordinal,
+ int numberOfTuples) throws HyracksDataException;
+
+ protected abstract void onNext() throws HyracksDataException;
+
+ @Override
+ public final void next() throws HyracksDataException {
+ onNext();
+ tupleIndex++;
+ }
+
+ @Override
+ public final void consume() {
+ tupleIndex = frame.getTupleCount();
+ }
+
+ @Override
+ public final boolean isConsumed() {
+ return tupleIndex >= frame.getTupleCount();
+ }
+
+ @Override
+ public final int getComponentIndex() {
+ return componentIndex;
+ }
+
+ @Override
+ public final void unpinColumnsPages() throws HyracksDataException {
+ for (int i = 0; i < buffersProviders.length; i++) {
+ buffersProviders[i].releaseAll();
+ }
+ }
+
+ @Override
+ public final void close() {
+ if (LOGGER.isInfoEnabled() && numOfSkippedMegaLeafNodes > 0) {
+ LOGGER.info("Filtered {} disk mega-leaf nodes out of {} in total", numOfSkippedMegaLeafNodes,
+ totalNumberOfMegaLeafNodes);
+ }
+ }
+
+ /* *************************************************************
+ * Unsupported Operations
+ * *************************************************************
+ */
+
+ @Override
+ public final void setFieldCount(int fieldCount) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void setFieldCount(int fieldStartIndex, int fieldCount) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void resetByTupleOffset(byte[] buf, int tupleStartOffset) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+
+ @Override
+ public final void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
+ throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnAwareDiskOnlyMultiComparator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnAwareDiskOnlyMultiComparator.java
new file mode 100644
index 0000000..297b740
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnAwareDiskOnlyMultiComparator.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples;
+
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.common.MultiComparator;
+
+public class ColumnAwareDiskOnlyMultiComparator extends MultiComparator {
+ public ColumnAwareDiskOnlyMultiComparator(MultiComparator comparator) {
+ super(comparator.getComparators());
+ }
+
+ @Override
+ public int compare(ITupleReference tupleA, ITupleReference tupleB) {
+ return ((IColumnTupleIterator) tupleA).compareTo((IColumnTupleIterator) tupleB);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnAwareMultiComparator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnAwareMultiComparator.java
new file mode 100644
index 0000000..0438f41
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnAwareMultiComparator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
+import org.apache.hyracks.storage.common.MultiComparator;
+
+public class ColumnAwareMultiComparator extends MultiComparator {
+ public ColumnAwareMultiComparator(IBinaryComparator[] cmps) {
+ super(cmps);
+ }
+
+ @Override
+ public int compare(ITupleReference tupleA, ITupleReference tupleB) throws HyracksDataException {
+ if (tupleA instanceof IColumnTupleIterator && tupleB instanceof IColumnTupleIterator) {
+ //Avoid comparing serialized data
+ return ((IColumnTupleIterator) tupleA).compareTo((IColumnTupleIterator) tupleB);
+ }
+ return super.compare(tupleA, tupleB);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiBufferProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiBufferProvider.java
new file mode 100644
index 0000000..0c17d6b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiBufferProvider.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Queue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+public final class ColumnMultiBufferProvider implements IColumnBufferProvider {
+ private final int columnIndex;
+ private final IColumnReadMultiPageOp multiPageOp;
+ private final Queue<ICachedPage> pages;
+ private int numberOfPages;
+ private int startPage;
+ private int startOffset;
+ private int length;
+
+ public ColumnMultiBufferProvider(int columnIndex, IColumnReadMultiPageOp multiPageOp) {
+ this.columnIndex = columnIndex;
+ this.multiPageOp = multiPageOp;
+ pages = new ArrayDeque<>();
+ }
+
+ @Override
+ public void reset(ColumnBTreeReadLeafFrame frame) throws HyracksDataException {
+ if (columnIndex >= frame.getNumberOfColumns()) {
+ numberOfPages = 0;
+ length = 0;
+ return;
+ }
+
+ int offset = frame.getColumnOffset(columnIndex);
+ startPage = frame.getPageId() + getColumnPageIndex(offset);
+ startOffset = offset % multiPageOp.getPageSize();
+ //Duplicate as the buffer could be shared by more than one column
+ ByteBuffer firstPage = readNext().duplicate();
+ firstPage.position(startOffset);
+ //Read the length
+ length = firstPage.getInt();
+ int remainingLength = length - firstPage.remaining();
+ numberOfPages = (int) Math.ceil((double) remainingLength / multiPageOp.getPageSize());
+ //+4-bytes after reading the length
+ startOffset += Integer.BYTES;
+ //-4-bytes after reading the length
+ length -= Integer.BYTES;
+ }
+
+ @Override
+ public void readAll(Queue<ByteBuffer> buffers) throws HyracksDataException {
+ ByteBuffer buffer = pages.peek().getBuffer().duplicate();
+ buffer.clear();
+ buffer.position(startOffset);
+ buffers.add(buffer);
+ for (int i = 0; i < numberOfPages; i++) {
+ buffer = readNext().duplicate();
+ buffer.clear();
+ buffers.add(buffer);
+ }
+ numberOfPages = 0;
+ }
+
+ @Override
+ public void releaseAll() throws HyracksDataException {
+ while (!pages.isEmpty()) {
+ ICachedPage page = pages.poll();
+ multiPageOp.unpin(page);
+ }
+ }
+
+ @Override
+ public int getLength() {
+ return length;
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ throw new UnsupportedOperationException("Use readAll() for multi-buffer");
+ }
+
+ @Override
+ public int getColumnIndex() {
+ return columnIndex;
+ }
+
+ private ByteBuffer readNext() throws HyracksDataException {
+ ICachedPage columnPage = multiPageOp.pin(startPage++);
+ pages.add(columnPage);
+ return columnPage.getBuffer();
+ }
+
+ private int getColumnPageIndex(int columnOffset) {
+ return (int) Math.floor((double) columnOffset / multiPageOp.getPageSize());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnSingleBufferProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnSingleBufferProvider.java
new file mode 100644
index 0000000..3ae5c7d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnSingleBufferProvider.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples;
+
+import java.nio.ByteBuffer;
+import java.util.Queue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+
+/**
+ * Buffer provider for Primary Keys
+ */
+public final class ColumnSingleBufferProvider implements IColumnBufferProvider {
+ private final int columnIndex;
+ private ByteBuffer buffer;
+
+ public ColumnSingleBufferProvider(int columnIndex) {
+ this.columnIndex = columnIndex;
+ }
+
+ @Override
+ public void reset(ColumnBTreeReadLeafFrame frame) {
+ int offset = frame.getColumnOffset(columnIndex);
+ this.buffer = frame.getBuffer().duplicate();
+ buffer.position(offset);
+ }
+
+ @Override
+ public void readAll(Queue<ByteBuffer> buffers) {
+ throw new UnsupportedOperationException("Use getBuffer() for single-buffer");
+ }
+
+ @Override
+ public void releaseAll() throws HyracksDataException {
+ //NoOp
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+
+ @Override
+ public int getLength() {
+ return -1;
+ }
+
+ @Override
+ public int getColumnIndex() {
+ return columnIndex;
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/ColumnUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/ColumnUtil.java
new file mode 100644
index 0000000..25ed3db
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/ColumnUtil.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.utils;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManager;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTree;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+
+public class ColumnUtil {
+ /**
+ * Used to get the columns info from {@link IComponentMetadata#get(IValueReference, ArrayBackedValueStorage)}
+ *
+ * @see LSMColumnBTree#activate()
+ * @see IColumnManager#activate(IValueReference)
+ */
+ private static final MutableArrayValueReference COLUMNS_METADATA_KEY =
+ new MutableArrayValueReference("COLUMNS_METADATA".getBytes());
+
+ private ColumnUtil() {
+ }
+
+ public static IValueReference getColumnMetadataCopy(IComponentMetadata src) throws HyracksDataException {
+ ArrayBackedValueStorage storage = new ArrayBackedValueStorage();
+ src.get(COLUMNS_METADATA_KEY, storage);
+ return storage;
+ }
+
+ public static void putColumnsMetadataValue(IValueReference columnsMetadataValue, IComponentMetadata dest)
+ throws HyracksDataException {
+ dest.put(COLUMNS_METADATA_KEY, columnsMetadataValue);
+ }
+
+ public static IColumnTupleProjector getTupleProjector(IIndexAccessParameters iap,
+ IColumnTupleProjector defaultProjector) {
+ IColumnTupleProjector projector =
+ iap.getParameter(HyracksConstants.TUPLE_PROJECTOR, IColumnTupleProjector.class);
+ return projector == null ? defaultProjector : projector;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
new file mode 100644
index 0000000..1a55447
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.utils;
+
+import java.util.List;
+
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManagerFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeLeafFrameFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTreeWithBloomFilterDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
+import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
+
+public class LSMColumnBTreeUtil {
+
+ public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ int[] btreeFields, IMetadataPageManagerFactory freePageManagerFactory, boolean updateAware, ITracer tracer,
+ ICompressorDecompressorFactory compressorDecompressorFactory, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector, IColumnManagerFactory columnManagerFactory)
+ throws HyracksDataException {
+
+ //Tuple writers
+ LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+ cmpFactories.length, false, updateAware, nullTypeTraits, nullIntrospector);
+ LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+ cmpFactories.length, true, updateAware, nullTypeTraits, nullIntrospector);
+ LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits,
+ cmpFactories.length, updateAware, nullTypeTraits, nullIntrospector);
+ LSMBTreeTupleWriterFactory bulkLoadTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+ cmpFactories.length, false, updateAware, nullTypeTraits, nullIntrospector);
+
+ //Leaf frames
+ ITreeIndexFrameFactory flushLeafFrameFactory = new ColumnBTreeLeafFrameFactory(copyTupleWriterFactory,
+ columnManagerFactory.getFlushColumnTupleReaderWriterFactory());
+ ITreeIndexFrameFactory mergeLeafFrameFactory = new ColumnBTreeLeafFrameFactory(copyTupleWriterFactory,
+ columnManagerFactory.createMergeColumnTupleReaderWriterFactory());
+ ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new ColumnBTreeLeafFrameFactory(bulkLoadTupleWriterFactory,
+ columnManagerFactory.getLoadColumnTupleReaderWriterFactory());
+ ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
+ ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
+
+ //BTree factory
+ TreeIndexFactory<ColumnBTree> flushBTreeFactory = new ColumnBTreeFactory(ioManager, diskBufferCache,
+ freePageManagerFactory, interiorFrameFactory, flushLeafFrameFactory, cmpFactories, typeTraits.length);
+ TreeIndexFactory<ColumnBTree> mergeBTreeFactory = new ColumnBTreeFactory(ioManager, diskBufferCache,
+ freePageManagerFactory, interiorFrameFactory, mergeLeafFrameFactory, cmpFactories, typeTraits.length);
+ TreeIndexFactory<ColumnBTree> bulkloadBTreeFactory =
+ new ColumnBTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory,
+ bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
+
+ ILSMIndexFileManager fileNameManager =
+ new LSMBTreeFileManager(ioManager, file, flushBTreeFactory, true, compressorDecompressorFactory);
+
+ BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ ILSMDiskComponentFactory flushComponentFactory =
+ new LSMColumnBTreeWithBloomFilterDiskComponentFactory(flushBTreeFactory, bloomFilterFactory);
+ ILSMDiskComponentFactory mergeComponentFactory =
+ new LSMColumnBTreeWithBloomFilterDiskComponentFactory(mergeBTreeFactory, bloomFilterFactory);
+ ILSMDiskComponentFactory bulkLoadComponentFactory =
+ new LSMColumnBTreeWithBloomFilterDiskComponentFactory(bulkloadBTreeFactory, bloomFilterFactory);
+
+ return new LSMColumnBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
+ deleteLeafFrameFactory, diskBufferCache, fileNameManager, flushComponentFactory, mergeComponentFactory,
+ bulkLoadComponentFactory, bloomFilterFalsePositiveRate, typeTraits.length, cmpFactories, mergePolicy,
+ opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, btreeFields, tracer,
+ columnManagerFactory.createColumnManager());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
index deafec3..e8b4aef 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorDescriptor.java
index 3800d17..3e03e5c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorDescriptor.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
public class LSMBTreeBatchPointSearchOperatorDescriptor extends BTreeSearchOperatorDescriptor {
private static final long serialVersionUID = 1L;
@@ -38,10 +39,11 @@
IIndexDataflowHelperFactory indexHelperFactory, boolean retainInput, boolean retainMissing,
IMissingWriterFactory missingWriterFactory, ISearchOperationCallbackFactory searchCallbackFactory,
int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, ITupleFilterFactory tupleFilterFactory,
- long outputLimit) {
+ long outputLimit, ITupleProjectorFactory tupleProjectorFactory) {
super(spec, outRecDesc, lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, minFilterFieldIndexes,
- maxFilterFieldIndexes, false, null, tupleFilterFactory, outputLimit, false, null, null);
+ maxFilterFieldIndexes, false, null, tupleFilterFactory, outputLimit, false, null, null,
+ tupleProjectorFactory);
}
@Override
@@ -51,7 +53,7 @@
recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), lowKeyFields, highKeyFields,
lowKeyInclusive, highKeyInclusive, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, tupleFilterFactory,
- outputLimit);
+ outputLimit, tupleProjectorFactory);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorNodePushable.java
index 30813ef..f6f97b7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeBatchPointSearchOperatorNodePushable.java
@@ -34,10 +34,12 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
public class LSMBTreeBatchPointSearchOperatorNodePushable extends BTreeSearchOperatorNodePushable {
@@ -48,18 +50,19 @@
boolean highKeyInclusive, int[] minFilterKeyFields, int[] maxFilterKeyFields,
IIndexDataflowHelperFactory indexHelperFactory, boolean retainInput, boolean retainMissing,
IMissingWriterFactory missingWriterFactory, ISearchOperationCallbackFactory searchCallbackFactory,
- ITupleFilterFactory tupleFilterFactory, long outputLimit) throws HyracksDataException {
+ ITupleFilterFactory tupleFilterFactory, long outputLimit, ITupleProjectorFactory tupleProjectorFactory)
+ throws HyracksDataException {
super(ctx, partition, inputRecDesc, lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive,
minFilterKeyFields, maxFilterKeyFields, indexHelperFactory, retainInput, retainMissing,
missingWriterFactory, searchCallbackFactory, false, null, tupleFilterFactory, outputLimit, false, null,
- null);
+ null, tupleProjectorFactory);
this.keyFields = lowKeyFields;
}
@Override
protected IIndexCursor createCursor() throws HyracksDataException {
ILSMIndexAccessor lsmAccessor = (ILSMIndexAccessor) indexAccessor;
- return new LSMBTreeBatchPointSearchCursor(lsmAccessor.getOpContext());
+ return ((LSMBTree) index).createBatchPointSearchCursor(lsmAccessor.getOpContext());
}
@Override
@@ -123,7 +126,7 @@
break;
}
}
- stats.getTupleCounter().update(matchingTupleCount);
+ stats.getInputTupleCounter().update(matchingTupleCount);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
index a7e433c..21c818c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
@@ -158,12 +158,12 @@
json.put("isSecondaryNoIncrementalMaintenance", isSecondaryNoIncrementalMaintenance);
}
- private static boolean getOrDefaultHasBloomFilter(JsonNode json, boolean isPrimary) {
+ protected static boolean getOrDefaultHasBloomFilter(JsonNode json, boolean isPrimary) {
// for backward compatibility, only primary indexes have bloom filters
return getOrDefaultBoolean(json, HAS_BLOOM_FILTER_FIELD, isPrimary);
}
- private static boolean getOrDefaultBoolean(JsonNode jsonNode, String fieldName, boolean defaultValue) {
+ protected static boolean getOrDefaultBoolean(JsonNode jsonNode, String fieldName, boolean defaultValue) {
return jsonNode.has(fieldName) ? jsonNode.get(fieldName).asBoolean() : defaultValue;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index c3d1416..d2fbbef 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -341,7 +341,7 @@
try {
List<ILSMComponent> mergedComponents = mergeOp.getMergingComponents();
long numElements = getNumberOfElements(mergedComponents);
- mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), null,
+ mergedComponent = createDiskComponent(getMergeComponentFactory(), mergeOp.getTarget(), null,
mergeOp.getBloomFilterTarget(), true);
IPageWriteCallback pageWriteCallback = pageWriteCallbackFactory.createPageWriteCallback();
componentBulkLoader = mergedComponent.createBulkLoader(operation, 1.0f, false, numElements, false,
@@ -418,7 +418,7 @@
}
public ILSMIndexAccessor createAccessor(AbstractLSMIndexOperationContext opCtx) {
- return new LSMTreeIndexAccessor(getHarness(), opCtx, cursorFactory);
+ return new LSMTreeIndexAccessor(getHarness(), opCtx, getCursorFactory());
}
@Override
@@ -483,8 +483,28 @@
returnDeletedTuples = true;
}
IIndexCursorStats stats = new IndexCursorStats();
- LSMBTreeRangeSearchCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples, stats);
+ LSMBTreeRangeSearchCursor cursor = createCursor(opCtx, returnDeletedTuples, stats);
return new LSMBTreeMergeOperation(accessor, cursor, stats, mergeFileRefs.getInsertIndexFileReference(),
mergeFileRefs.getBloomFilterFileReference(), callback, getIndexIdentifier());
}
+
+ public LSMBTreeBatchPointSearchCursor createBatchPointSearchCursor(ILSMIndexOperationContext opCtx) {
+ return new LSMBTreeBatchPointSearchCursor(opCtx);
+ }
+
+ protected LSMBTreeRangeSearchCursor createCursor(AbstractLSMIndexOperationContext opCtx,
+ boolean returnDeletedTuples, IIndexCursorStats stats) {
+ return new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples, stats);
+ }
+
+ /**
+ * @return Merge component factory (could be different from {@link #componentFactory}
+ */
+ protected ILSMDiskComponentFactory getMergeComponentFactory() {
+ return componentFactory;
+ }
+
+ protected ICursorFactory getCursorFactory() {
+ return cursorFactory;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBatchPointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBatchPointSearchCursor.java
index 8ab6fb1..73d06d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBatchPointSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBatchPointSearchCursor.java
@@ -21,14 +21,15 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import org.apache.hyracks.storage.am.btree.impls.BatchPredicate;
-import org.apache.hyracks.storage.am.btree.impls.DiskBTreePointSearchCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
/**
* This cursor performs point searches for each batch of search keys.
* Assumption: the search keys must be sorted into the increasing order.
- *
*/
public class LSMBTreeBatchPointSearchCursor extends LSMBTreePointSearchCursor {
@@ -42,7 +43,9 @@
while (!foundTuple && batchPred.hasNext()) {
batchPred.next();
if (foundIn >= 0) {
- btreeCursors[foundIn].close();
+ if (operationalComponents.get(foundIn).getType() == LSMComponentType.MEMORY) {
+ btreeCursors[foundIn].close();
+ }
foundIn = -1;
}
foundTuple = super.doHasNext();
@@ -56,6 +59,11 @@
}
@Override
+ protected ITreeIndexCursor createCursor(LSMComponentType type, BTreeAccessor btreeAccessor) {
+ return btreeAccessor.createPointCursor(false, true);
+ }
+
+ @Override
protected boolean isSearchCandidate(int componentIndex) throws HyracksDataException {
if (!super.isSearchCandidate(componentIndex)) {
return false;
@@ -68,21 +76,6 @@
maxFileterKey, opCtx.getFilterCmp());
}
- @Override
- protected void closeCursors() throws HyracksDataException {
- super.closeCursors();
- if (btreeCursors != null) {
- // clear search states of btree cursors
- for (int i = 0; i < numBTrees; ++i) {
- if (btreeCursors[i] != null) {
- if (btreeCursors[i] instanceof DiskBTreePointSearchCursor) {
- ((DiskBTreePointSearchCursor) btreeCursors[i]).clearSearchState();
- }
- }
- }
- }
- }
-
public int getKeyIndex() {
return ((BatchPredicate) predicate).getKeyIndex();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
index 1312e30..a00e10e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
@@ -43,7 +43,7 @@
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.util.trace.ITracer;
-public final class LSMBTreeOpContext extends AbstractLSMIndexOperationContext {
+public class LSMBTreeOpContext extends AbstractLSMIndexOperationContext {
/*
* Finals
@@ -74,9 +74,9 @@
IBinaryComparatorFactory[] filterCmpFactories, ITracer tracer) {
super(index, btreeFields, filterFields, filterCmpFactories, searchCallback, modificationCallback, tracer);
LSMBTreeMemoryComponent c = (LSMBTreeMemoryComponent) mutableComponents.get(0);
- IBinaryComparatorFactory cmpFactories[] = c.getIndex().getComparatorFactories();
+ IBinaryComparatorFactory[] cmpFactories = c.getIndex().getComparatorFactories();
if (cmpFactories[0] != null) {
- this.cmp = MultiComparator.create(c.getIndex().getComparatorFactories());
+ this.cmp = createMultiComparator(c.getIndex().getComparatorFactories());
} else {
this.cmp = null;
}
@@ -112,6 +112,10 @@
insertSearchCursor = new LSMBTreePointSearchCursor(this);
}
+ protected MultiComparator createMultiComparator(IBinaryComparatorFactory[] cmpFactories) {
+ return MultiComparator.create(cmpFactories);
+ }
+
@Override
public void setOperation(IndexOperation newOp) {
reset();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
index d4903d9..9740bc4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
@@ -199,7 +199,8 @@
for (int i = 0; i < numBTrees; i++) {
ILSMComponent component = operationalComponents.get(i);
BTree btree = (BTree) component.getIndex();
- if (component.getType() == LSMComponentType.MEMORY) {
+ LSMComponentType type = component.getType();
+ if (type == LSMComponentType.MEMORY) {
includeMutableComponent = true;
if (bloomFilters[i] != null) {
destroyAndNullifyCursorAtIndex(i);
@@ -212,8 +213,8 @@
}
if (btreeAccessors[i] == null) {
- btreeAccessors[i] = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
- btreeCursors[i] = btreeAccessors[i].createPointCursor(false, false);
+ btreeAccessors[i] = createAccessor(type, btree, i);
+ btreeCursors[i] = createCursor(type, btreeAccessors[i]);
} else {
// re-use
btreeAccessors[i].reset(btree, NoOpIndexAccessParameters.INSTANCE);
@@ -225,6 +226,14 @@
hashComputed = false;
}
+ protected BTreeAccessor createAccessor(LSMComponentType type, BTree btree, int i) throws HyracksDataException {
+ return btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ }
+
+ protected ITreeIndexCursor createCursor(LSMComponentType type, BTreeAccessor btreeAccessor) {
+ return btreeAccessor.createPointCursor(false, false);
+ }
+
private void destroyAndNullifyCursorAtIndex(int i) throws HyracksDataException {
// component at location i was a disk component before, and is now a memory component, or vise versa
bloomFilters[i] = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
index 2c5fb50..968416c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
@@ -49,7 +49,7 @@
private final RangePredicate reusablePred;
private ISearchOperationCallback searchCallback;
private BTreeAccessor[] btreeAccessors;
- private boolean[] isMemoryComponent;
+ protected boolean[] isMemoryComponent;
private ArrayTupleBuilder tupleBuilder;
private boolean canCallProceed = true;
private boolean resultOfSearchCallbackProceed = false;
@@ -149,6 +149,7 @@
// There are no more elements in the memory component.. can safely skip locking for the
// remaining operations
includeMutableComponent = false;
+ excludeMemoryComponent();
}
}
} else {
@@ -180,6 +181,7 @@
// the tree of head tuple
// the head element of PQ is useless now
PriorityQueueElement e = outputPriorityQueue.poll();
+ markAsDeleted(e);
pushIntoQueueFromCursorAndReplaceThisElement(e);
} else {
// If the previous tuple and the head tuple are different
@@ -200,6 +202,14 @@
}
+ protected void excludeMemoryComponent() {
+ //NoOp
+ }
+
+ protected void markAsDeleted(PriorityQueueElement e) throws HyracksDataException {
+ //NoOp
+ }
+
private void pushOutputElementIntoQueueIfNeeded() throws HyracksDataException {
if (needPushElementIntoQueue) {
pushIntoQueueFromCursorAndReplaceThisElement(outputElement);
@@ -250,7 +260,7 @@
}
}
- private int replaceFrom() throws HyracksDataException {
+ protected int replaceFrom() throws HyracksDataException {
int replaceFrom = -1;
if (!switchPossible) {
return replaceFrom;
@@ -386,20 +396,21 @@
}
for (int i = 0; i < numBTrees; i++) {
ILSMComponent component = operationalComponents.get(i);
+ LSMComponentType type = component.getType();
BTree btree;
if (component.getType() == LSMComponentType.MEMORY) {
includeMutableComponent = true;
}
btree = (BTree) component.getIndex();
if (btreeAccessors[i] == null || destroyIncompatible(component, i)) {
- btreeAccessors[i] = btree.createAccessor(iap);
- rangeCursors[i] = btreeAccessors[i].createSearchCursor(false);
+ btreeAccessors[i] = createAccessor(type, btree, i);
+ rangeCursors[i] = createCursor(type, btreeAccessors[i]);
} else {
// re-use
btreeAccessors[i].reset(btree, iap);
rangeCursors[i].close();
}
- isMemoryComponent[i] = component.getType() == LSMComponentType.MEMORY;
+ isMemoryComponent[i] = type == LSMComponentType.MEMORY;
}
IndexCursorUtils.open(btreeAccessors, rangeCursors, searchPred);
try {
@@ -433,4 +444,12 @@
return resultOfSearchCallbackProceed;
}
+ protected BTreeAccessor createAccessor(LSMComponentType type, BTree btree, int index) throws HyracksDataException {
+ return btree.createAccessor(iap);
+ }
+
+ protected IIndexCursor createCursor(LSMComponentType type, BTreeAccessor accessor) {
+ return accessor.createSearchCursor(false);
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java
index efacad1..aa72267 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java
@@ -46,6 +46,13 @@
scanCursor = new LSMBTreeDiskComponentScanCursor(opCtx);
}
+ protected LSMBTreeSearchCursor(LSMBTreePointSearchCursor pointCursor, LSMBTreeRangeSearchCursor rangeCursor,
+ LSMBTreeDiskComponentScanCursor scanCursor) {
+ this.pointCursor = pointCursor;
+ this.rangeCursor = rangeCursor;
+ this.scanCursor = scanCursor;
+ }
+
@Override
public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMBTreeCursorInitialState lsmInitialState = (LSMBTreeCursorInitialState) initialState;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
index 7226f3b..d8d155e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexBulkLoader.java
index 13a0e27..acb84e1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexBulkLoader.java
@@ -20,7 +20,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex.AbstractTreeIndexBulkLoader;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndexBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index 27875c0..2c97221 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -236,7 +236,7 @@
} else {
// If the previous tuple and the head tuple are different
// the info of previous tuple is useless
- if (needPushElementIntoQueue == true) {
+ if (needPushElementIntoQueue) {
pushIntoQueueFromCursorAndReplaceThisElement(outputElement);
needPushElementIntoQueue = false;
}
@@ -304,6 +304,10 @@
public MultiComparator getMultiComparator() {
return cmp;
}
+
+ public void setMultiComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
}
protected void setPriorityQueueComparator() {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
index 3a58d74..19b3f25 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>hyracks</artifactId>
<groupId>org.apache.hyracks</groupId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
index 7d3eff1..7e88d1f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
index b7eb115..11385de 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.ILSMIndexCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -50,7 +51,7 @@
protected boolean open;
protected RTreeSearchCursor[] rtreeCursors;
- protected BTreeRangeSearchCursor[] btreeCursors;
+ protected ITreeIndexCursor[] btreeCursors;
protected RTreeAccessor[] rtreeAccessors;
protected BTreeAccessor[] btreeAccessors;
protected BloomFilter[] bloomFilters;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 8e5cb35..729ca74 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -142,7 +142,8 @@
bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBuddyIndex().getFileId(), linearizerArray,
btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(),
flushingComponent.getBuddyIndex().getBufferCache(), comparatorFields);
- BTreeRangeSearchCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
+ BTreeRangeSearchCursor btreeScanCursor =
+ (BTreeRangeSearchCursor) memBTreeAccessor.createSearchCursor(false);
try {
isEmpty = true;
memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/pom.xml
index 0c92622..80f71fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
index a8a4252..e1c6f5b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
+import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
import org.apache.hyracks.storage.am.rtree.util.RTreeUtils;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
@@ -59,7 +60,7 @@
super(ctx, inputRecDesc, partition, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, appendIndexFilter,
nonFilterWriterFactory, null, -1, appendOpCallbackProceedResult, searchCallbackProceedResultFalseValue,
- searchCallbackProceedResultTrueValue);
+ searchCallbackProceedResultTrueValue, DefaultTupleProjectorFactory.INSTANCE);
if (keyFields != null && keyFields.length > 0) {
searchKey = new PermutingFrameTupleReference();
searchKey.setFieldPermutation(keyFields);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index 7e8f249..d85200f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -41,6 +41,7 @@
import org.apache.hyracks.storage.am.common.frames.AbstractSlotManager;
import org.apache.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndexBulkLoader;
import org.apache.hyracks.storage.am.common.impls.NodeFrontier;
import org.apache.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
@@ -902,7 +903,7 @@
return new RTreeBulkLoader(fillFactor, callback);
}
- public class RTreeBulkLoader extends AbstractTreeIndex.AbstractTreeIndexBulkLoader {
+ public class RTreeBulkLoader extends AbstractTreeIndexBulkLoader {
ITreeIndexFrame lowerFrame, prevInteriorFrame;
RTreeTypeAwareTupleWriter interiorFrameTupleWriter =
((RTreeTypeAwareTupleWriter) interiorFrame.getTupleWriter());
@@ -911,7 +912,7 @@
List<Integer> prevNodeFrontierPages = new ArrayList<>();
public RTreeBulkLoader(float fillFactor, IPageWriteCallback callback) throws HyracksDataException {
- super(fillFactor, callback);
+ super(fillFactor, callback, RTree.this);
prevInteriorFrame = interiorFrameFactory.createFrame();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml
index a1c4b02..076f0c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
<license>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessParameters.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessParameters.java
index a3745d0..8f553d9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessParameters.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessParameters.java
@@ -40,4 +40,14 @@
* Gets additional parameters.
*/
Map<String, Object> getParameters();
+
+ /**
+ * Gets a parameter.
+ *
+ * @param key of a parameter
+ * @param clazz used to explicitly cast the requested parameter to the required type
+ * @param <T> the required type
+ * @return the requested parameter
+ */
+ <T> T getParameter(String key, Class<T> clazz);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index f1fe86f..5578d27 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -552,6 +552,10 @@
BufferedFileHandle fInfo = getFileHandle(cPage);
cPage.buffer.clear();
fInfo.read(cPage);
+ final IThreadStats threadStats = statsSubscribers.get(Thread.currentThread());
+ if (threadStats != null) {
+ threadStats.coldRead();
+ }
}
@Override
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
index 3c1a24d..ba23e30 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
@@ -16,20 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.hyracks.storage.common.projection;
-/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
- */
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
+import java.io.DataOutput;
+import java.io.IOException;
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public interface ITupleProjector {
+ ITupleReference project(ITupleReference tuple, DataOutput dos, ArrayTupleBuilder tb) throws IOException;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjectorFactory.java
similarity index 62%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
copy to hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjectorFactory.java
index 3c1a24d..ff9ecf9 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionInfo.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjectorFactory.java
@@ -16,20 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.metadata;
+package org.apache.hyracks.storage.common.projection;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
/**
- * Generic interface to include the projection information for
- * {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
+ * Tuple projector allows the data source to project the values needed before it pushed up
+ * to the upper operator.
*/
-public interface IProjectionInfo<T> {
- /**
- * @return projected values' information
- */
- T getProjectionInfo();
-
- /**
- * @return a copy of the {@link IProjectionInfo}
- */
- IProjectionInfo<T> createCopy();
+public interface ITupleProjectorFactory extends Serializable {
+ ITupleProjector createTupleProjector(IHyracksTaskContext context) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml b/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml
index 8e4b23f..6bd4b01 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-dataflow-common-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-dataflow-common-test/pom.xml
index 06381a9..c402398 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-dataflow-common-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-dataflow-common-test/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/pom.xml
index f061079..bdc870c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
index dc93d8b..a866c47 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<properties>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
index 2e882ac..5c2613a 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/pom.xml
index b0ea9ab..c4b3a62 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
index 10f1705..f1bafb7 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>hyracks-tests</artifactId>
<groupId>org.apache.hyracks</groupId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
index 4629123..52415b9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
index 3462ec2..84b4960 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
index 53808b9..4e9da31 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
<plugins>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/pom.xml
index 00c7949..3b6c8f5 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<licenses>
diff --git a/hyracks-fullstack/hyracks/hyracks-util/pom.xml b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
index 7653ad9..ffe5549 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hyracks</artifactId>
<groupId>org.apache.hyracks</groupId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
index dc63ac4..e782a69 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
@@ -31,4 +31,16 @@
* @return the pinned pages count
*/
long getPinnedPagesCount();
+
+ /**
+ * Indicates that this thread caused a cold read from disk
+ */
+ void coldRead();
+
+ /**
+ * Gets the count of pages read in from disk
+ *
+ * @return the cold read count
+ */
+ long getColdReadCount();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
index c79eefc..9af3fb1 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
@@ -26,6 +26,7 @@
public class ThreadStats implements IThreadStats {
private AtomicLong pinnedPagesCount = new AtomicLong();
+ private AtomicLong coldReadCount = new AtomicLong();
@Override
public void pagePinned() {
@@ -36,4 +37,14 @@
public long getPinnedPagesCount() {
return pinnedPagesCount.get();
}
+
+ @Override
+ public long getColdReadCount() {
+ return coldReadCount.get();
+ }
+
+ @Override
+ public void coldRead() {
+ coldReadCount.incrementAndGet();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
index 3eb8687..cde79cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
@@ -221,6 +221,10 @@
return VarLenIntEncoderDecoder.decode(b, s);
}
+ public static int getNumBytesToStoreLength(byte[] bytes, int start) {
+ return getNumBytesToStoreLength(getUTFLength(bytes, start));
+ }
+
public static int getNumBytesToStoreLength(int strlen) {
return VarLenIntEncoderDecoder.getBytesRequired(strlen);
}
@@ -244,12 +248,13 @@
* consistent with the comparison result.
*/
public static int normalize(byte[] bytes, int start) {
- int len = getUTFLength(bytes, start);
long nk = 0;
+ int len = getUTFLength(bytes, start);
int offset = start + getNumBytesToStoreLength(len);
+ int end = offset + len;
for (int i = 0; i < 2; ++i) {
nk <<= 16;
- if (i < len) {
+ if (offset < end) {
nk += (charAt(bytes, offset)) & 0xffff;
offset += charSize(bytes, offset);
}
@@ -498,19 +503,15 @@
* are exactly the same as for the <code>readUTF</code>
* method of <code>DataInput</code>.
*
- * @param in
- * a data input stream.
+ * @param in a data input stream.
* @return a Unicode string.
- * @throws EOFException
- * if the input stream reaches the end
- * before all the bytes.
- * @throws IOException
- * the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @throws UTFDataFormatException
- * if the bytes do not represent a
- * valid modified UTF-8 encoding of a Unicode string.
+ * @throws EOFException if the input stream reaches the end
+ * before all the bytes.
+ * @throws IOException the stream has been closed and the contained
+ * input stream does not support reading after close, or
+ * another I/O error occurs.
+ * @throws UTFDataFormatException if the bytes do not represent a
+ * valid modified UTF-8 encoding of a Unicode string.
* @see java.io.DataInputStream#readUnsignedShort()
*/
public static String readUTF8(DataInput in) throws IOException {
@@ -602,10 +603,8 @@
/**
* Write a UTF8 String <code>str</code> into the DataOutput <code>out</code>
*
- * @param str,
- * a Unicode string;
- * @param out,
- * a Data output stream.
+ * @param str, a Unicode string;
+ * @param out, a Data output stream.
* @throws IOException
*/
public static void writeUTF8(CharSequence str, DataOutput out) throws IOException {
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java
index b114351..eb3a5b6 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java
@@ -35,7 +35,8 @@
public static final String STRING_LEN_3 = "xyz";
public static final String STRING_UTF8_3 = "锟斤拷";
- public static final String STRING_UTF8_MIX = "\uD841\uDF0E\uD841\uDF31锟X斤Y拷Zà"; // one, two, three, and four bytes
+ // one, two, three, and four bytes
+ public static final String STRING_UTF8_MIX = "\uD841\uDF0E\uD841\uDF31锟X斤Y拷Zà";
public static final String STRING_UTF8_MIX_LOWERCASE = "\uD841\uDF0E\uD841\uDF31锟x斤y拷zà";
public static final String STRING_NEEDS_2_JAVA_CHARS_1 = "\uD83D\uDE22\uD83D\uDE22\uD83D\uDC89\uD83D\uDC89";
public static final String STRING_NEEDS_2_JAVA_CHARS_2 = "😢😢💉💉";
@@ -44,6 +45,8 @@
public static final String STRING_EMOJI_FAMILY_OF_2 = "\uD83D\uDC68\u200D\uD83D\uDC66";
public static final String EMOJI_BASKETBALL = "\uD83C\uDFC0";
+ public static final String THREE_BYTES_UTF8_CHAR = "ह";
+
public static final String STRING_LEN_127 = generateStringRepeatBy(ONE_ASCII_CHAR, 127);
public static final String STRING_LEN_128 = generateStringRepeatBy(ONE_ASCII_CHAR, 128);
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
index c7468d2..4eb1fc3 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
@@ -25,6 +25,7 @@
import static org.apache.hyracks.util.string.UTF8StringSample.STRING_UTF8_3;
import static org.apache.hyracks.util.string.UTF8StringSample.STRING_UTF8_MIX;
import static org.apache.hyracks.util.string.UTF8StringSample.STRING_UTF8_MIX_LOWERCASE;
+import static org.apache.hyracks.util.string.UTF8StringSample.THREE_BYTES_UTF8_CHAR;
import static org.apache.hyracks.util.string.UTF8StringUtil.charAt;
import static org.apache.hyracks.util.string.UTF8StringUtil.charSize;
import static org.apache.hyracks.util.string.UTF8StringUtil.compareTo;
@@ -77,13 +78,14 @@
}
@Test
- public void testCompareToAndNormolize() throws Exception {
+ public void testCompareToAndNormalize() throws Exception {
testCompare(STRING_UTF8_MIX, STRING_UTF8_MIX, OPTION.STANDARD);
testCompare(STRING_UTF8_3, STRING_UTF8_MIX, OPTION.STANDARD);
testCompare(STRING_LEN_MEDIUM, STRING_UTF8_MIX, OPTION.STANDARD);
+ testCompare(THREE_BYTES_UTF8_CHAR, THREE_BYTES_UTF8_CHAR, OPTION.STANDARD);
}
- public boolean isSameSign(int r1, int r2) {
+ private static boolean isSameSign(int r1, int r2) {
if (r1 > 0) {
return r2 > 0;
}
@@ -99,7 +101,7 @@
LOWERCASE
}
- public void testCompare(String str1, String str2, OPTION option) throws IOException {
+ private static void testCompare(String str1, String str2, OPTION option) {
byte[] buffer1 = writeStringToBytes(str1);
byte[] buffer2 = writeStringToBytes(str2);
@@ -117,7 +119,6 @@
assertEquals(str1.compareToIgnoreCase(str2), lowerCaseCompareTo(buffer1, 0, buffer2, 0));
break;
}
-
}
@Test
diff --git a/hyracks-fullstack/hyracks/pom.xml b/hyracks-fullstack/hyracks/pom.xml
index 443e8c0..a35a61e 100644
--- a/hyracks-fullstack/hyracks/pom.xml
+++ b/hyracks-fullstack/hyracks/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>apache-hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
</parent>
<build>
@@ -88,6 +88,7 @@
<module>hyracks-storage-am-lsm-invertedindex</module>
<module>hyracks-storage-am-lsm-common</module>
<module>hyracks-storage-am-lsm-btree</module>
+ <module>hyracks-storage-am-lsm-btree-column</module>
<module>hyracks-storage-am-lsm-rtree</module>
<module>hyracks-storage-am-rtree</module>
<module>hyracks-test-support</module>
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 695b749..348b5bf 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hyracks</groupId>
<artifactId>apache-hyracks</artifactId>
- <version>0.3.8-SNAPSHOT</version>
+ <version>0.3.9-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hyracks-ecosystem-full-stack</name>
<url>${implementation.url}</url>
diff --git a/pom.xml b/pom.xml
index eac0b88..dcdd73e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>apache-asterixdb-fullstack</artifactId>
- <version>0.9.8-SNAPSHOT</version>
+ <version>0.9.9-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hyracks-asterix</name>