Merge branch 'gerrit/mad-hatter' into 'master'
Change-Id: I2059c989bd17c0e1677269ae48af3fd38ccce12b
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InjectTypeCastForFunctionArgumentsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InjectTypeCastForFunctionArgumentsRule.java
index 77d63f1..9531e94 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InjectTypeCastForFunctionArgumentsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InjectTypeCastForFunctionArgumentsRule.java
@@ -144,7 +144,8 @@
new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CAST_TYPE),
new ArrayList<>(Collections.singletonList(new MutableObject<>(argExpr))));
castFunc.setSourceLocation(argExpr.getSourceLocation());
- TypeCastUtils.setRequiredAndInputTypes(castFunc, funcOutputType, type);
+ IAType funcOutputPrimeType = TypeComputeUtils.getActualType(funcOutputType);
+ TypeCastUtils.setRequiredAndInputTypes(castFunc, funcOutputPrimeType, type, false);
argRef.setValue(castFunc);
return true;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
index c186687..924c679 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
@@ -52,8 +52,10 @@
private Map<Index, Integer> indexNumMatchedKeys = new TreeMap<Index, Integer>();
// variables for resetting null placeholder for left-outer-join
- private Mutable<ILogicalOperator> lojGroupbyOpRef = null;
- private ScalarFunctionCallExpression lojIsMissingFuncInGroupBy = null;
+ // See AccessMethodUtils#removeUnjoinedDuplicatesInLOJ() for a definition of a special GroupBy
+ // and extra output processing steps needed when it's not available.
+ private Mutable<ILogicalOperator> lojSpecialGroupByOpRef = null;
+ private ScalarFunctionCallExpression lojIsMissingFuncInSpecialGroupBy = null;
// For a secondary index, if we use only PK and secondary key field in a plan, it is an index-only plan.
// Contains information about index-only plan
@@ -134,20 +136,20 @@
indexNumMatchedKeys.put(index, numMatchedKeys);
}
- public void setLOJGroupbyOpRef(Mutable<ILogicalOperator> opRef) {
- lojGroupbyOpRef = opRef;
+ public void setLOJSpecialGroupByOpRef(Mutable<ILogicalOperator> opRef) {
+ lojSpecialGroupByOpRef = opRef;
}
- public Mutable<ILogicalOperator> getLOJGroupbyOpRef() {
- return lojGroupbyOpRef;
+ public Mutable<ILogicalOperator> getLOJSpecialGroupByOpRef() {
+ return lojSpecialGroupByOpRef;
}
- public void setLOJIsMissingFuncInGroupBy(ScalarFunctionCallExpression isMissingFunc) {
- lojIsMissingFuncInGroupBy = isMissingFunc;
+ public void setLOJIsMissingFuncInSpecialGroupBy(ScalarFunctionCallExpression isMissingFunc) {
+ lojIsMissingFuncInSpecialGroupBy = isMissingFunc;
}
- public ScalarFunctionCallExpression getLOJIsMissingFuncInGroupBy() {
- return lojIsMissingFuncInGroupBy;
+ public ScalarFunctionCallExpression getLOJIsMissingFuncInSpecialGroupBy() {
+ return lojIsMissingFuncInSpecialGroupBy;
}
public Dataset getDatasetFromIndexDatasetMap(Index idx) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 10037f0..1357fd2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -93,6 +93,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.WindowOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
@@ -717,29 +718,63 @@
*/
public static boolean finalizeJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree indexSubTree,
- AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin,
- boolean hasGroupBy, ILogicalOperator indexSearchOp, LogicalVariable newNullPlaceHolderVar,
+ OptimizableOperatorSubTree probeSubTree, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean isLeftOuterJoinWithSpecialGroupBy,
+ ILogicalOperator indexSearchOp, LogicalVariable newNullPlaceHolderVar,
Mutable<ILogicalExpression> conditionRef, Dataset dataset) throws AlgebricksException {
+ boolean isIndexOnlyPlan = analysisCtx.getIndexOnlyPlanInfo().getFirst();
+ List<LogicalVariable> probePKVars = null;
ILogicalOperator finalIndexSearchOp = indexSearchOp;
- if (isLeftOuterJoin && hasGroupBy) {
- ScalarFunctionCallExpression lojFuncExprs = analysisCtx.getLOJIsMissingFuncInGroupBy();
- List<LogicalVariable> lojMissingVariables = new ArrayList<>();
- lojFuncExprs.getUsedVariables(lojMissingVariables);
- boolean lojMissingVarExist = false;
- if (!lojMissingVariables.isEmpty()) {
- lojMissingVarExist = true;
+ if (isLeftOuterJoin) {
+ if (isLeftOuterJoinWithSpecialGroupBy) {
+ ScalarFunctionCallExpression lojFuncExprs = analysisCtx.getLOJIsMissingFuncInSpecialGroupBy();
+ List<LogicalVariable> lojMissingVariables = new ArrayList<>();
+ lojFuncExprs.getUsedVariables(lojMissingVariables);
+ boolean lojMissingVarExist = !lojMissingVariables.isEmpty();
+
+ // Resets the missing place holder variable.
+ AccessMethodUtils.resetLOJMissingPlaceholderVarInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+
+ // For the index-only plan, if newNullPlaceHolderVar is not in the variable map of the union operator
+ // or if the variable is removed during the above method, we need to refresh the variable mapping in UNION.
+ if (isIndexOnlyPlan) {
+ finalIndexSearchOp = AccessMethodUtils.resetVariableMappingInUnionOpInIndexOnlyPlan(
+ lojMissingVarExist, lojMissingVariables, indexSearchOp, afterJoinRefs, context);
+ }
+ } else {
+ // We'll need to remove unjoined duplicates after the left outer join if there is no special GroupBy,
+ // but in order to do that we need to know the keys of the probe branch.
+ // If probe keys are not available then we fail this transformation
+ // See AccessMethodUtils#removeUnjoinedDuplicatesInLOJ() for a definition of a special GroupBy
+ // and extra output processing steps needed when it's not available.
+ if (probeSubTree.hasDataSource()) {
+ probePKVars = new ArrayList<>();
+ probeSubTree.getPrimaryKeyVars(null, probePKVars);
+ }
+ if (probePKVars == null || probePKVars.isEmpty()) {
+ return false;
+ }
+ if (isIndexOnlyPlan) {
+ // re-map probe branch keys after UnionAll introduced by the indexonly plan
+ if (indexSearchOp.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
+ return false;
+ }
+ UnionAllOperator unionAllOp = (UnionAllOperator) indexSearchOp;
+ for (int i = 0, ln = probePKVars.size(); i < ln; i++) {
+ LogicalVariable unionAllOutputVar = findUnionAllOutputVariable(unionAllOp, probePKVars.get(i));
+ if (unionAllOutputVar == null) {
+ return false;
+ }
+ probePKVars.set(i, unionAllOutputVar);
+ }
+ }
}
-
- // Resets the missing place holder variable.
- AccessMethodUtils.resetLOJMissingPlaceholderVarInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
-
- // For the index-only plan, if newNullPlaceHolderVar is not in the variable map of the union operator
- // or if the variable is removed during the above method, we need to refresh the variable mapping in UNION.
- finalIndexSearchOp = AccessMethodUtils.resetVariableMappingInUnionOpInIndexOnlyPlan(lojMissingVarExist,
- lojMissingVariables, indexSearchOp, afterJoinRefs, context);
}
- boolean isIndexOnlyPlan = analysisCtx.getIndexOnlyPlanInfo().getFirst();
+ SourceLocation sourceLoc = joinRef.getValue().getSourceLocation();
+
+ ILogicalOperator finalOp;
+
// If there are any left conditions, add a new select operator on top.
indexSubTree.getDataSourceRef().setValue(finalIndexSearchOp);
if (conditionRef.getValue() != null) {
@@ -754,27 +789,105 @@
indexSubTree.getDataSourceRef().setValue(dataSourceRefOp);
}
// Replaces the current operator with the newly created UNIONALL operator.
- joinRef.setValue(finalIndexSearchOp);
+ finalOp = finalIndexSearchOp;
} else {
// Non-index only plan case
indexSubTree.getDataSourceRef().setValue(finalIndexSearchOp);
SelectOperator topSelectOp = new SelectOperator(conditionRef, isLeftOuterJoin, newNullPlaceHolderVar);
- topSelectOp.setSourceLocation(finalIndexSearchOp.getSourceLocation());
+ topSelectOp.setSourceLocation(sourceLoc);
topSelectOp.getInputs().add(indexSubTree.getRootRef());
topSelectOp.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(topSelectOp);
- joinRef.setValue(topSelectOp);
+ finalOp = topSelectOp;
}
} else {
if (finalIndexSearchOp.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
- joinRef.setValue(finalIndexSearchOp);
+ finalOp = finalIndexSearchOp;
} else {
- joinRef.setValue(indexSubTree.getRootRef().getValue());
+ finalOp = indexSubTree.getRootRef().getValue();
}
}
+
+ if (isLeftOuterJoin && !isLeftOuterJoinWithSpecialGroupBy) {
+ finalOp = removeUnjoinedDuplicatesInLOJ(finalOp, probePKVars, newNullPlaceHolderVar, context, sourceLoc);
+ }
+
+ joinRef.setValue(finalOp);
return true;
}
+ /**
+ * In case of a left outer join we look for a special GroupBy above the join operator
+ * (see {@link IntroduceJoinAccessMethodRule#checkAndApplyJoinTransformation(Mutable, IOptimizationContext)}.
+ * A "Special GroupBy" is a GroupBy that eliminates unjoined duplicates that might be produced by the secondary
+ * index probe. We probe secondary indexes on each index partition and return a tuple with a right branch variable
+ * set to MISSING if there's no match on that partition. Therefore if there's more than one partition where
+ * nothing joined then the index probe plan will produce several such MISSING tuples, however the join result
+ * must have a single MISSING tuple for each unjoined left branch tuple. If the special GroupBy is available then
+ * it'll eliminate those MISSING duplicates, otherwise this method is called to produce additional operators to
+ * achieve that. The special GroupBy operators are introduced by the optimizer when rewriting FROM-LET or
+ * equivalent patterns into a left outer join with parent a group by.
+ * <p>
+ * The plan generated by this method to eliminate unjoined duplicates is as follows:
+ * <ul>
+ * <li> SelectOp $m</li>
+ * <li> WindowOp [$m=win-mark-first-missing-impl(right-branch-non-missing-value)]
+ * PARTITION BY left-branch-key
+ * ORDER BY right-branch-non-missing-value DESC</li>
+ * <li> input_subtree</li>
+ * </ul>
+ * <p>
+ * The {@link org.apache.asterix.runtime.runningaggregates.std.WinMarkFirstMissingRunningAggregateDescriptor
+ * win-mark-first-missing-impl} internal function assigns {@code TRUE} for each tuple that has a non-MISSING
+ * value that comes from the right branch or the first MISSING value in the window partition. The remaining
+ * tuples in each window partition are unjoined duplicate tuples and will be assigned {@code FALSE}. Then
+ * the Select operator eliminates those unjoined duplicate tuples.
+ */
+ private static SelectOperator removeUnjoinedDuplicatesInLOJ(ILogicalOperator inputOp,
+ List<LogicalVariable> probePKVars, LogicalVariable newNullPlaceHolderVar, IOptimizationContext context,
+ SourceLocation sourceLoc) throws AlgebricksException {
+ if (probePKVars == null || probePKVars.isEmpty()) {
+ throw new IllegalArgumentException();
+ }
+ List<Mutable<ILogicalExpression>> winPartitionByList = new ArrayList<>(probePKVars.size());
+ for (LogicalVariable probePKeyVar : probePKVars) {
+ VariableReferenceExpression probePKeyVarRef = new VariableReferenceExpression(probePKeyVar);
+ probePKeyVarRef.setSourceLocation(sourceLoc);
+ winPartitionByList.add(new MutableObject<>(probePKeyVarRef));
+ }
+
+ VariableReferenceExpression winOrderByVarRef = new VariableReferenceExpression(newNullPlaceHolderVar);
+ winOrderByVarRef.setSourceLocation(sourceLoc);
+ /* Sort in DESC order, so all MISSING values are at the end */
+ Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> winOrderByPair =
+ new Pair<>(OrderOperator.DESC_ORDER, new MutableObject<>(winOrderByVarRef));
+
+ LogicalVariable winVar = context.newVar();
+ VariableReferenceExpression winOrderByVarRef2 = new VariableReferenceExpression(newNullPlaceHolderVar);
+ winOrderByVarRef2.setSourceLocation(sourceLoc);
+ AbstractFunctionCallExpression winExpr =
+ BuiltinFunctions.makeWindowFunctionExpression(BuiltinFunctions.WIN_MARK_FIRST_MISSING_IMPL,
+ Collections.singletonList(new MutableObject<>(winOrderByVarRef2)));
+
+ WindowOperator winOp = new WindowOperator(winPartitionByList, Collections.singletonList(winOrderByPair));
+ winOp.getVariables().add(winVar);
+ winOp.getExpressions().add(new MutableObject<>(winExpr));
+ winOp.getInputs().add(new MutableObject<>(inputOp));
+ winOp.setExecutionMode(ExecutionMode.PARTITIONED);
+ winOp.setSourceLocation(sourceLoc);
+ context.computeAndSetTypeEnvironmentForOperator(winOp);
+
+ VariableReferenceExpression winVarRef = new VariableReferenceExpression(winVar);
+ winVarRef.setSourceLocation(sourceLoc);
+ SelectOperator selectOp = new SelectOperator(new MutableObject<>(winVarRef), false, null);
+ selectOp.getInputs().add(new MutableObject<>(winOp));
+ selectOp.setExecutionMode(ExecutionMode.LOCAL);
+ selectOp.setSourceLocation(sourceLoc);
+ context.computeAndSetTypeEnvironmentForOperator(selectOp);
+
+ return selectOp;
+ }
+
public static ILogicalOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType,
ARecordType metaRecordType, Index index, ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams,
IOptimizationContext context, boolean retainInput, boolean retainNull,
@@ -1583,27 +1696,21 @@
ALogicalPlanImpl subPlan = (ALogicalPlanImpl) lojGroupbyOp.getNestedPlans().get(0);
Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0);
AbstractLogicalOperator subPlanRootOp = (AbstractLogicalOperator) subPlanRootOpRef.getValue();
- ScalarFunctionCallExpression isMissingFuncExpr = findIsMissingInSubplan(subPlanRootOp, rightSubTree);
-
- if (isMissingFuncExpr == null) {
- throw CompilationException.create(ErrorCode.CANNOT_FIND_NON_MISSING_SELECT_OPERATOR,
- lojGroupbyOp.getSourceLocation());
- }
- return isMissingFuncExpr;
+ return findIsMissingInSubplan(subPlanRootOp, rightSubTree);
}
public static void resetLOJMissingPlaceholderVarInGroupByOp(AccessMethodAnalysisContext analysisCtx,
LogicalVariable newMissingPlaceholderVaraible, IOptimizationContext context) throws AlgebricksException {
//reset the missing placeholder variable in groupby operator
- ScalarFunctionCallExpression isMissingFuncExpr = analysisCtx.getLOJIsMissingFuncInGroupBy();
+ ScalarFunctionCallExpression isMissingFuncExpr = analysisCtx.getLOJIsMissingFuncInSpecialGroupBy();
isMissingFuncExpr.getArguments().clear();
VariableReferenceExpression newMissingVarRef = new VariableReferenceExpression(newMissingPlaceholderVaraible);
newMissingVarRef.setSourceLocation(isMissingFuncExpr.getSourceLocation());
isMissingFuncExpr.getArguments().add(new MutableObject<ILogicalExpression>(newMissingVarRef));
//recompute type environment.
- OperatorPropertiesUtil.typeOpRec(analysisCtx.getLOJGroupbyOpRef(), context);
+ OperatorPropertiesUtil.typeOpRec(analysisCtx.getLOJSpecialGroupByOpRef(), context);
}
// New < For external datasets indexing>
@@ -2558,4 +2665,15 @@
return AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION_DEFAULT_VALUE;
}
+ /**
+ * Finds an output variable for the given input variable of UnionAllOperator.
+ */
+ static LogicalVariable findUnionAllOutputVariable(UnionAllOperator unionAllOp, LogicalVariable inputVar) {
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : unionAllOp.getVariableMappings()) {
+ if (t.first.equals(inputVar) || t.second.equals(inputVar)) {
+ return t.third;
+ }
+ }
+ return null;
+ }
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 85e0c72..0bcc202 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -239,7 +239,8 @@
public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree,
OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
- IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException {
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean isLeftOuterJoinWithSpecialGroupBy)
+ throws AlgebricksException {
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
Mutable<ILogicalExpression> conditionRef = joinOp.getCondition();
@@ -268,6 +269,7 @@
if (isLeftOuterJoin) {
// Gets a new null place holder variable that is the first field variable of the primary key
// from the indexSubTree's datasourceScanOp.
+ // We need this for all left outer joins, even those that do not have a special GroupBy
newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
}
@@ -285,8 +287,9 @@
return false;
}
- return AccessMethodUtils.finalizeJoinPlanTransformation(afterJoinRefs, joinRef, indexSubTree, analysisCtx,
- context, isLeftOuterJoin, hasGroupBy, indexSearchOp, newNullPlaceHolderVar, conditionRef, dataset);
+ return AccessMethodUtils.finalizeJoinPlanTransformation(afterJoinRefs, joinRef, indexSubTree, probeSubTree,
+ analysisCtx, context, isLeftOuterJoin, isLeftOuterJoinWithSpecialGroupBy, indexSearchOp,
+ newNullPlaceHolderVar, conditionRef, dataset);
}
/**
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
index 559f336..94de169 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
@@ -104,7 +104,8 @@
public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree,
OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
- IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException;
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean isLeftOuterJoinWithSpecialGroupBy)
+ throws AlgebricksException;
/**
* Analyzes expr to see whether it is optimizable by the given concrete index.
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
index 577754a..199f878 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
@@ -43,6 +43,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
/**
@@ -87,7 +88,6 @@
protected final OptimizableOperatorSubTree leftSubTree = new OptimizableOperatorSubTree();
protected final OptimizableOperatorSubTree rightSubTree = new OptimizableOperatorSubTree();
protected IVariableTypeEnvironment typeEnvironment = null;
- protected boolean hasGroupBy = true;
protected List<Mutable<ILogicalOperator>> afterJoinRefs = null;
// Registers access methods.
@@ -191,22 +191,10 @@
}
/**
- * Checks whether the given operator is LEFTOUTERJOIN.
- * If so, also checks that GROUPBY is placed after LEFTOUTERJOIN.
+ * Checks whether the given operator has a single child which is a LEFTOUTERJOIN.
*/
- // Check whether (Groupby)? <-- Leftouterjoin
- private boolean isLeftOuterJoin(AbstractLogicalOperator op1) {
- if (op1.getInputs().size() != 1) {
- return false;
- }
- if (op1.getInputs().get(0).getValue().getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
- return false;
- }
- if (op1.getOperatorTag() == LogicalOperatorTag.GROUP) {
- return true;
- }
- hasGroupBy = false;
- return true;
+ private int findLeftOuterJoinChild(AbstractLogicalOperator op) {
+ return OperatorManipulationUtil.findChild(op, LogicalOperatorTag.LEFTOUTERJOIN);
}
/**
@@ -254,35 +242,37 @@
// Now, we are sure that transformation attempts for earlier joins have been failed.
// Checks the current operator pattern to see whether it is a JOIN or not.
- boolean isThisOpInnerJoin = isInnerJoin(op);
- boolean isThisOpLeftOuterJoin = isLeftOuterJoin(op);
- boolean isParentOpGroupBy = hasGroupBy;
+ boolean isInnerJoin = false;
+ boolean isLeftOuterJoin = false;
+ int leftOuterJoinChildIdx;
Mutable<ILogicalOperator> joinRefFromThisOp = null;
AbstractBinaryJoinOperator joinOpFromThisOp = null;
// operators that need to be removed from the afterJoinRefs list.
Mutable<ILogicalOperator> opRefRemove = opRef;
- if (isThisOpInnerJoin) {
- // Sets the join operator.
+ if (isInnerJoin(op)) {
+ // Sets the inner join operator.
+ isInnerJoin = true;
joinRef = opRef;
joinOp = (InnerJoinOperator) op;
joinRefFromThisOp = opRef;
joinOpFromThisOp = (InnerJoinOperator) op;
- } else if (isThisOpLeftOuterJoin) {
+ } else if ((leftOuterJoinChildIdx = findLeftOuterJoinChild(op)) >= 0) {
// Sets the left-outer-join operator.
- // The current operator is GROUP and the child of this operator is LEFTOUERJOIN.
- joinRef = op.getInputs().get(0);
+ // A child of the current operator is LEFTOUTERJOIN.
+ isLeftOuterJoin = true;
+ joinRef = op.getInputs().get(leftOuterJoinChildIdx);
joinOp = (LeftOuterJoinOperator) joinRef.getValue();
- joinRefFromThisOp = op.getInputs().get(0);
+ joinRefFromThisOp = op.getInputs().get(leftOuterJoinChildIdx);
joinOpFromThisOp = (LeftOuterJoinOperator) joinRefFromThisOp.getValue();
-
- // Group-by should not be removed at this point since the given left-outer-join can be transformed.
- opRefRemove = op.getInputs().get(0);
+ // Left outer join's parent operator should not be removed at this point since the given left-outer-join
+ // can be transformed.
+ opRefRemove = op.getInputs().get(leftOuterJoinChildIdx);
}
afterJoinRefs.remove(opRefRemove);
// For a JOIN case, tries to transform the given plan.
- if (isThisOpInnerJoin || isThisOpLeftOuterJoin) {
+ if (isInnerJoin || isLeftOuterJoin) {
// Restores the information from this operator since it might have been be set to null
// if there are other join operators in the earlier path.
@@ -375,13 +365,24 @@
// Applies the plan transformation using chosen index.
AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first);
- // For LOJ with GroupBy, prepare objects to reset LOJ nullPlaceHolderVariable
- // in GroupByOp.
- if (isThisOpLeftOuterJoin && isParentOpGroupBy) {
- analysisCtx.setLOJGroupbyOpRef(opRef);
- ScalarFunctionCallExpression isNullFuncExpr = AccessMethodUtils
- .findLOJIsMissingFuncInGroupBy((GroupByOperator) opRef.getValue(), rightSubTree);
- analysisCtx.setLOJIsMissingFuncInGroupBy(isNullFuncExpr);
+ // For a left outer join with a special GroupBy, prepare objects to reset LOJ's
+ // nullPlaceHolderVariable in that GroupBy's nested plan.
+ // See AccessMethodUtils#removeUnjoinedDuplicatesInLOJ() for a definition of a special GroupBy
+ // and extra output processing steps needed when it's not available.
+ boolean isLeftOuterJoinWithSpecialGroupBy;
+ if (isLeftOuterJoin && op.getOperatorTag() == LogicalOperatorTag.GROUP) {
+ GroupByOperator groupByOp = (GroupByOperator) opRef.getValue();
+ ScalarFunctionCallExpression isNullFuncExpr =
+ AccessMethodUtils.findLOJIsMissingFuncInGroupBy(groupByOp, rightSubTree);
+ // TODO:(dmitry) do we need additional checks to ensure that this is a special GroupBy,
+ // i.e. that this GroupBy will eliminate unjoined duplicates?
+ isLeftOuterJoinWithSpecialGroupBy = isNullFuncExpr != null;
+ if (isLeftOuterJoinWithSpecialGroupBy) {
+ analysisCtx.setLOJSpecialGroupByOpRef(opRef);
+ analysisCtx.setLOJIsMissingFuncInSpecialGroupBy(isNullFuncExpr);
+ }
+ } else {
+ isLeftOuterJoinWithSpecialGroupBy = false;
}
Dataset indexDataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex.second);
@@ -396,8 +397,8 @@
// Finally, tries to apply plan transformation using the chosen index.
boolean res = chosenIndex.first.applyJoinPlanTransformation(afterJoinRefs, joinRef, leftSubTree,
- rightSubTree, chosenIndex.second, analysisCtx, context, isThisOpLeftOuterJoin,
- isParentOpGroupBy);
+ rightSubTree, chosenIndex.second, analysisCtx, context, isLeftOuterJoin,
+ isLeftOuterJoinWithSpecialGroupBy);
// If the plan transformation is successful, we don't need to traverse the plan
// any more, since if there are more JOIN operators, the next trigger on this plan
@@ -412,10 +413,10 @@
joinOp = null;
}
- // Checked the given left-outer-join operator and it is not transformed. So, this group-by operator
- // after the left-outer-join operator should be removed from the afterJoinRefs list
- // since the current operator is a group-by operator.
- if (isThisOpLeftOuterJoin) {
+ // Checked the given left-outer-join operator and it is not transformed.
+ // So, the left-outer-join's parent operator should be removed from the afterJoinRefs list
+ // since the current operator is that parent operator.
+ if (isLeftOuterJoin) {
afterJoinRefs.remove(opRef);
}
@@ -425,8 +426,6 @@
/**
* After the pattern is matched, checks the condition and initializes the data source
* from the right (inner) sub tree.
- *
- * @throws AlgebricksException
*/
protected boolean checkJoinOpConditionAndInitSubTree(IOptimizationContext context) throws AlgebricksException {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index ed73333..ca65a9e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -497,7 +497,8 @@
public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree,
OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
- IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException {
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean isLeftOuterJoinWithSpecialGroupBy)
+ throws AlgebricksException {
Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex);
OptimizableOperatorSubTree indexSubTree;
OptimizableOperatorSubTree probeSubTree;
@@ -524,13 +525,16 @@
//if LOJ, reset null place holder variable
LogicalVariable newNullPlaceHolderVar = null;
- if (isLeftOuterJoin && hasGroupBy) {
+ if (isLeftOuterJoin) {
//get a new null place holder variable that is the first field variable of the primary key
//from the indexSubTree's datasourceScanOp
+ // We need this for all left outer joins, even those that do not have a special GroupBy
newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
- //reset the null place holder variable
- AccessMethodUtils.resetLOJMissingPlaceholderVarInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+ if (isLeftOuterJoinWithSpecialGroupBy) {
+ //reset the null place holder variable
+ AccessMethodUtils.resetLOJMissingPlaceholderVarInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+ }
}
AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) joinRef.getValue();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 19558aa..7a24a8b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -306,7 +306,8 @@
public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree,
OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
- IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException {
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean isLeftOuterJoinWithSpecialGroupBy)
+ throws AlgebricksException {
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
Mutable<ILogicalExpression> conditionRef = joinOp.getCondition();
@@ -335,6 +336,7 @@
if (isLeftOuterJoin) {
// Gets a new null place holder variable that is the first field variable of the primary key
// from the indexSubTree's datasourceScanOp.
+ // We need this for all left outer joins, even those that do not have a special GroupBy
newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
}
@@ -352,8 +354,9 @@
return false;
}
- return AccessMethodUtils.finalizeJoinPlanTransformation(afterJoinRefs, joinRef, indexSubTree, analysisCtx,
- context, isLeftOuterJoin, hasGroupBy, indexSearchOp, newNullPlaceHolderVar, conditionRef, dataset);
+ return AccessMethodUtils.finalizeJoinPlanTransformation(afterJoinRefs, joinRef, indexSubTree, probeSubTree,
+ analysisCtx, context, isLeftOuterJoin, isLeftOuterJoinWithSpecialGroupBy, indexSearchOp,
+ newNullPlaceHolderVar, conditionRef, dataset);
}
@Override
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index 89ffcbe..3fdcc33 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -24,9 +24,13 @@
import java.io.Serializable;
import java.rmi.RemoteException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.asterix.common.api.IResponsePrinter;
import org.apache.asterix.common.exceptions.ACIDException;
@@ -62,15 +66,32 @@
/**
* Results are returned with the first response
*/
- IMMEDIATE,
+ IMMEDIATE("immediate"),
/**
* Results are produced completely, but only a result handle is returned
*/
- DEFERRED,
+ DEFERRED("deferred"),
/**
* A result handle is returned before the resutlts are complete
*/
- ASYNC
+ ASYNC("async");
+
+ private static final Map<String, ResultDelivery> deliveryNames =
+ Collections.unmodifiableMap(Arrays.stream(ResultDelivery.values())
+ .collect(Collectors.toMap(ResultDelivery::getName, Function.identity())));
+ private final String name;
+
+ ResultDelivery(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public static ResultDelivery fromName(String name) {
+ return deliveryNames.get(name);
+ }
}
class ResultMetadata implements Serializable {
@@ -88,15 +109,32 @@
private static final long serialVersionUID = 5885273238208454611L;
public enum ProfileType {
- COUNTS,
- FULL
+ COUNTS("counts"),
+ FULL("timings"),
+ NONE("off");
+
+ private static final Map<String, ProfileType> profileNames = Collections.unmodifiableMap(Arrays
+ .stream(ProfileType.values()).collect(Collectors.toMap(ProfileType::getName, Function.identity())));
+ private final String name;
+
+ ProfileType(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public static ProfileType fromName(String name) {
+ return profileNames.get(name);
+ }
}
private long count;
private long size;
private long processedObjects;
private Profile profile;
- private ProfileType type;
+ private ProfileType profileType;
private long totalWarningsCount;
public long getCount() {
@@ -141,12 +179,12 @@
return profile != null ? profile.getProfile() : null;
}
- public ProfileType getType() {
- return type;
+ public ProfileType getProfileType() {
+ return profileType;
}
- public void setType(ProfileType type) {
- this.type = type;
+ public void setProfileType(ProfileType profileType) {
+ this.profileType = profileType;
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/CcQueryCancellationServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/CcQueryCancellationServlet.java
index 00aef9e..7ba2867 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/CcQueryCancellationServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/CcQueryCancellationServlet.java
@@ -21,7 +21,7 @@
import java.io.IOException;
import java.util.concurrent.ConcurrentMap;
-import org.apache.asterix.api.http.server.QueryServiceServlet.Parameter;
+import org.apache.asterix.api.http.server.QueryServiceRequestParameters.Parameter;
import org.apache.asterix.common.api.IClientRequest;
import org.apache.asterix.common.api.IRequestTracker;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
index 053e042..617c937 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
@@ -24,7 +24,7 @@
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
-import org.apache.asterix.api.http.server.QueryServiceServlet.Parameter;
+import org.apache.asterix.api.http.server.QueryServiceRequestParameters.Parameter;
import org.apache.asterix.app.message.CancelQueryRequest;
import org.apache.asterix.app.message.CancelQueryResponse;
import org.apache.asterix.common.messaging.api.INCMessageBroker;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index acd6784..6953d1f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -32,7 +32,6 @@
import org.apache.asterix.app.message.ExecuteStatementResponseMessage;
import org.apache.asterix.app.result.ResponsePrinter;
import org.apache.asterix.app.result.fields.NcResultPrinter;
-import org.apache.asterix.common.api.Duration;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.api.IRequestReference;
import org.apache.asterix.common.config.GlobalConfig;
@@ -82,17 +81,14 @@
MessageFuture responseFuture = ncMb.registerMessageFuture();
final String handleUrl = getHandleUrl(param.getHost(), param.getPath(), delivery);
try {
- long timeout = ExecuteStatementRequestMessage.DEFAULT_NC_TIMEOUT_MILLIS;
- if (param.getTimeout() != null && !param.getTimeout().trim().isEmpty()) {
- timeout = TimeUnit.NANOSECONDS.toMillis(Duration.parseDurationStringToNanos(param.getTimeout()));
- }
+ long timeout = param.getTimeout();
int stmtCategoryRestrictionMask = org.apache.asterix.app.translator.RequestParameters
.getStatementCategoryRestrictionMask(param.isReadOnly());
- ExecuteStatementRequestMessage requestMsg =
- new ExecuteStatementRequestMessage(ncCtx.getNodeId(), responseFuture.getFutureId(), queryLanguage,
- statementsText, sessionOutput.config(), resultProperties.getNcToCcResultProperties(),
- param.getClientContextID(), handleUrl, optionalParameters, statementParameters,
- param.isMultiStatement(), param.isProfile(), stmtCategoryRestrictionMask, requestReference);
+ ExecuteStatementRequestMessage requestMsg = new ExecuteStatementRequestMessage(ncCtx.getNodeId(),
+ responseFuture.getFutureId(), queryLanguage, statementsText, sessionOutput.config(),
+ resultProperties.getNcToCcResultProperties(), param.getClientContextID(), handleUrl,
+ optionalParameters, statementParameters, param.isMultiStatement(), param.getProfileType(),
+ stmtCategoryRestrictionMask, requestReference);
execution.start();
ncMb.sendMessageToPrimaryCC(requestMsg);
try {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
index 9664e80..6e70d27 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
@@ -19,42 +19,119 @@
package org.apache.asterix.api.http.server;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import org.apache.asterix.common.api.Duration;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.translator.IStatementExecutor.ResultDelivery;
+import org.apache.asterix.translator.IStatementExecutor.Stats.ProfileType;
+import org.apache.asterix.translator.SessionConfig.OutputFormat;
+import org.apache.asterix.translator.SessionConfig.PlanFormat;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.http.HttpHeaders;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.server.utils.HttpUtil;
import org.apache.hyracks.util.JSONUtil;
+import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.ImmutableMap;
public class QueryServiceRequestParameters {
- public static final long DEFAULT_MAX_WARNINGS = 0L;
+ public enum Parameter {
+ ARGS("args"),
+ STATEMENT("statement"),
+ FORMAT("format"),
+ CLIENT_ID("client_context_id"),
+ PRETTY("pretty"),
+ MODE("mode"),
+ TIMEOUT("timeout"),
+ PLAN_FORMAT("plan-format"),
+ MAX_RESULT_READS("max-result-reads"),
+ EXPRESSION_TREE("expression-tree"),
+ REWRITTEN_EXPRESSION_TREE("rewritten-expression-tree"),
+ LOGICAL_PLAN("logical-plan"),
+ OPTIMIZED_LOGICAL_PLAN("optimized-logical-plan"),
+ PARSE_ONLY("parse-only"),
+ READ_ONLY("readonly"),
+ JOB("job"),
+ PROFILE("profile"),
+ SIGNATURE("signature"),
+ MULTI_STATEMENT("multi-statement"),
+ MAX_WARNINGS("max-warnings");
+
+ private final String str;
+
+ Parameter(String str) {
+ this.str = str;
+ }
+
+ public String str() {
+ return str;
+ }
+ }
+
+ private enum Attribute {
+ HEADER("header"),
+ LOSSLESS("lossless");
+
+ private final String str;
+
+ Attribute(String str) {
+ this.str = str;
+ }
+
+ public String str() {
+ return str;
+ }
+ }
+
+ private static final Map<String, PlanFormat> planFormats = ImmutableMap.of(HttpUtil.ContentType.JSON,
+ PlanFormat.JSON, "clean_json", PlanFormat.JSON, "string", PlanFormat.STRING);
+ private static final Map<String, Boolean> booleanValues =
+ ImmutableMap.of(Boolean.TRUE.toString(), Boolean.TRUE, Boolean.FALSE.toString(), Boolean.FALSE);
+ private static final Map<String, Boolean> csvHeaderValues =
+ ImmutableMap.of("present", Boolean.TRUE, "absent", Boolean.FALSE);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private String host;
private String path;
private String statement;
- private String format;
- private String timeout;
- private boolean pretty;
private String clientContextID;
- private String mode;
- private String maxResultReads;
- private String planFormat;
- private Map<String, JsonNode> statementParams;
- private boolean expressionTree;
- private boolean parseOnly; //don't execute; simply check for syntax correctness and named parameters.
- private boolean readOnly; // only allow statements that belong to QUERY category, fail for all other categories.
- private boolean rewrittenExpressionTree;
- private boolean logicalPlan;
- private boolean optimizedLogicalPlan;
- private boolean job;
- private boolean profile;
- private boolean signature;
- private boolean multiStatement;
- private long maxWarnings = DEFAULT_MAX_WARNINGS;
+ private OutputFormat format = OutputFormat.CLEAN_JSON;
+ private ResultDelivery mode = ResultDelivery.IMMEDIATE;
+ private PlanFormat planFormat = PlanFormat.JSON;
+ private ProfileType profileType = ProfileType.COUNTS;
+ private Map<String, JsonNode> statementParams = null;
+ private boolean pretty = false;
+ private boolean expressionTree = false;
+ private boolean parseOnly = false; // don't execute; simply check for syntax correctness and named parameters.
+ private boolean readOnly = false; // only allow statements belonging to QUERY category, fail for other categories.
+ private boolean rewrittenExpressionTree = false;
+ private boolean logicalPlan = false;
+ private boolean optimizedLogicalPlan = false;
+ private boolean job = false;
+ private boolean isCSVWithHeader = false;
+ private boolean signature = true;
+ private boolean multiStatement = true;
+ private long timeout = TimeUnit.MILLISECONDS.toMillis(Long.MAX_VALUE);
+ private long maxResultReads = 1L;
+ private long maxWarnings = 0L;
public String getHost() {
return host;
@@ -80,19 +157,24 @@
this.statement = statement;
}
- public String getFormat() {
+ public OutputFormat getFormat() {
return format;
}
- public void setFormat(String format) {
- this.format = format;
+ public void setFormat(Pair<OutputFormat, Boolean> formatAndHeader) {
+ Objects.requireNonNull(formatAndHeader);
+ Objects.requireNonNull(formatAndHeader.getLeft());
+ this.format = formatAndHeader.getLeft();
+ if (format == OutputFormat.CSV) {
+ isCSVWithHeader = formatAndHeader.getRight();
+ }
}
- public String getTimeout() {
+ public long getTimeout() {
return timeout;
}
- public void setTimeout(String timeout) {
+ public void setTimeout(long timeout) {
this.timeout = timeout;
}
@@ -112,27 +194,29 @@
this.clientContextID = clientContextID;
}
- public String getMode() {
+ public ResultDelivery getMode() {
return mode;
}
- public void setMode(String mode) {
+ public void setMode(ResultDelivery mode) {
+ Objects.requireNonNull(mode);
this.mode = mode;
}
- public String getMaxResultReads() {
+ public long getMaxResultReads() {
return maxResultReads;
}
- public void setMaxResultReads(String maxResultReads) {
+ public void setMaxResultReads(long maxResultReads) {
this.maxResultReads = maxResultReads;
}
- public String getPlanFormat() {
+ public PlanFormat getPlanFormat() {
return planFormat;
}
- public void setPlanFormat(String planFormat) {
+ public void setPlanFormat(PlanFormat planFormat) {
+ Objects.requireNonNull(planFormat);
this.planFormat = planFormat;
}
@@ -200,12 +284,17 @@
this.job = job;
}
- public void setProfile(boolean profile) {
- this.profile = profile;
+ public void setProfileType(ProfileType profileType) {
+ Objects.requireNonNull(profileType);
+ this.profileType = profileType;
}
- public boolean isProfile() {
- return profile;
+ public ProfileType getProfileType() {
+ return profileType;
+ }
+
+ public boolean isCSVWithHeader() {
+ return format == OutputFormat.CSV && isCSVWithHeader;
}
public boolean isSignature() {
@@ -238,18 +327,18 @@
object.put("path", path);
object.put("statement", statement != null ? JSONUtil.escape(new StringBuilder(), statement).toString() : null);
object.put("pretty", pretty);
- object.put("mode", mode);
+ object.put("mode", mode.getName());
object.put("clientContextID", clientContextID);
- object.put("format", format);
+ object.put("format", format.toString());
object.put("timeout", timeout);
object.put("maxResultReads", maxResultReads);
- object.put("planFormat", planFormat);
+ object.put("planFormat", planFormat.toString());
object.put("expressionTree", expressionTree);
object.put("rewrittenExpressionTree", rewrittenExpressionTree);
object.put("logicalPlan", logicalPlan);
object.put("optimizedLogicalPlan", optimizedLogicalPlan);
object.put("job", job);
- object.put("profile", profile);
+ object.put("profile", profileType.getName());
object.put("signature", signature);
object.put("multiStatement", multiStatement);
object.put("parseOnly", parseOnly);
@@ -272,4 +361,266 @@
return e.toString();
}
}
+
+ public void setParameters(QueryServiceServlet servlet, IServletRequest request, Map<String, String> optionalParams)
+ throws IOException {
+ setHost(servlet.host(request));
+ setPath(servlet.servletPath(request));
+ String contentType = HttpUtil.getContentTypeOnly(request);
+ try {
+ if (HttpUtil.ContentType.APPLICATION_JSON.equals(contentType)) {
+ setParamFromJSON(request, optionalParams);
+ } else {
+ setParamFromRequest(request, optionalParams);
+ }
+ } catch (JsonParseException | JsonMappingException e) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_JSON_VAL);
+ }
+ }
+
+ private void setParamFromJSON(IServletRequest request, Map<String, String> optionalParameters) throws IOException {
+ JsonNode jsonRequest = OBJECT_MAPPER.readTree(HttpUtil.getRequestBody(request));
+ setParams(jsonRequest, request.getHeader(HttpHeaders.ACCEPT), QueryServiceRequestParameters::getParameter);
+ setStatementParams(getOptStatementParameters(jsonRequest, jsonRequest.fieldNames(), JsonNode::get, v -> v));
+ setJsonOptionalParameters(jsonRequest, optionalParameters);
+ }
+
+ private void setParamFromRequest(IServletRequest request, Map<String, String> optionalParameters)
+ throws IOException {
+ setParams(request, request.getHeader(HttpHeaders.ACCEPT), IServletRequest::getParameter);
+ setStatementParams(getOptStatementParameters(request, request.getParameterNames().iterator(),
+ IServletRequest::getParameter, OBJECT_MAPPER::readTree));
+ setOptionalParameters(request, optionalParameters);
+ }
+
+ private <Req> void setParams(Req req, String acceptHeader, BiFunction<Req, String, String> valGetter)
+ throws HyracksDataException {
+ setStatement(valGetter.apply(req, Parameter.STATEMENT.str()));
+ setClientContextID(valGetter.apply(req, Parameter.CLIENT_ID.str()));
+
+ setFormatIfExists(req, acceptHeader, Parameter.FORMAT.str(), valGetter);
+ setMode(parseIfExists(req, Parameter.MODE.str(), valGetter, getMode(), ResultDelivery::fromName));
+ setPlanFormat(parseIfExists(req, Parameter.PLAN_FORMAT.str(), valGetter, getPlanFormat(), planFormats::get));
+ setProfileType(parseIfExists(req, Parameter.PROFILE.str(), valGetter, getProfileType(), ProfileType::fromName));
+
+ setTimeout(parseTime(req, Parameter.TIMEOUT.str(), valGetter, getTimeout()));
+ setMaxResultReads(parseLong(req, Parameter.MAX_RESULT_READS.str(), valGetter, getMaxResultReads()));
+ setMaxWarnings(parseLong(req, Parameter.MAX_WARNINGS.str(), valGetter, getMaxWarnings()));
+
+ setPretty(parseBoolean(req, Parameter.PRETTY.str(), valGetter, isPretty()));
+ setExpressionTree(parseBoolean(req, Parameter.EXPRESSION_TREE.str(), valGetter, isExpressionTree()));
+ setRewrittenExpressionTree(
+ parseBoolean(req, Parameter.REWRITTEN_EXPRESSION_TREE.str(), valGetter, isRewrittenExpressionTree()));
+ setLogicalPlan(parseBoolean(req, Parameter.LOGICAL_PLAN.str(), valGetter, isLogicalPlan()));
+ setParseOnly(parseBoolean(req, Parameter.PARSE_ONLY.str(), valGetter, isParseOnly()));
+ setReadOnly(parseBoolean(req, Parameter.READ_ONLY.str(), valGetter, isReadOnly()));
+ setOptimizedLogicalPlan(
+ parseBoolean(req, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), valGetter, isOptimizedLogicalPlan()));
+ setMultiStatement(parseBoolean(req, Parameter.MULTI_STATEMENT.str(), valGetter, isMultiStatement()));
+ setJob(parseBoolean(req, Parameter.JOB.str(), valGetter, isJob()));
+ setSignature(parseBoolean(req, Parameter.SIGNATURE.str(), valGetter, isSignature()));
+ }
+
+ protected void setJsonOptionalParameters(JsonNode jsonRequest, Map<String, String> optionalParameters)
+ throws HyracksDataException {
+ // allows extensions to set extra parameters
+ }
+
+ protected void setOptionalParameters(IServletRequest request, Map<String, String> optionalParameters)
+ throws HyracksDataException {
+ // allows extensions to set extra parameters
+ }
+
+ @FunctionalInterface
+ interface CheckedFunction<I, O> {
+ O apply(I requestParamValue) throws IOException;
+ }
+
+ private <R, P> Map<String, JsonNode> getOptStatementParameters(R request, Iterator<String> paramNameIter,
+ BiFunction<R, String, P> paramValueAccessor, CheckedFunction<P, JsonNode> paramValueParser)
+ throws IOException {
+ Map<String, JsonNode> result = null;
+ while (paramNameIter.hasNext()) {
+ String paramName = paramNameIter.next();
+ String stmtParamName = extractStatementParameterName(paramName);
+ if (stmtParamName != null) {
+ if (result == null) {
+ result = new HashMap<>();
+ }
+ P paramValue = paramValueAccessor.apply(request, paramName);
+ JsonNode stmtParamValue = paramValueParser.apply(paramValue);
+ result.put(stmtParamName, stmtParamValue);
+ } else if (Parameter.ARGS.str().equals(toLower(paramName))) {
+ if (result == null) {
+ result = new HashMap<>();
+ }
+ P paramValue = paramValueAccessor.apply(request, paramName);
+ JsonNode stmtParamValue = paramValueParser.apply(paramValue);
+ if (!stmtParamValue.isArray()) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, paramName, stmtParamValue.asText());
+ }
+ for (int i = 0, ln = stmtParamValue.size(); i < ln; i++) {
+ result.put(String.valueOf(i + 1), stmtParamValue.get(i));
+ }
+ }
+ }
+ return result;
+ }
+
+ public static String extractStatementParameterName(String name) {
+ int ln = name.length();
+ if ((ln == 2 || isStatementParameterNameRest(name, 2)) && name.charAt(0) == '$'
+ && Character.isLetter(name.charAt(1))) {
+ return name.substring(1);
+ }
+ return null;
+ }
+
+ private static boolean isStatementParameterNameRest(CharSequence input, int startIndex) {
+ int i = startIndex;
+ for (int ln = input.length(); i < ln; i++) {
+ char c = input.charAt(i);
+ boolean ok = c == '_' || Character.isLetterOrDigit(c);
+ if (!ok) {
+ return false;
+ }
+ }
+ return i > startIndex;
+ }
+
+ private static <R> boolean parseBoolean(R request, String parameterName,
+ BiFunction<R, String, String> valueAccessor, boolean defaultVal) throws HyracksDataException {
+ String value = toLower(valueAccessor.apply(request, parameterName));
+ if (value == null) {
+ return defaultVal;
+ }
+ Boolean booleanVal = booleanValues.get(value);
+ if (booleanVal == null) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, parameterName, value);
+
+ }
+ return booleanVal.booleanValue();
+ }
+
+ private static <R> long parseLong(R request, String parameterName, BiFunction<R, String, String> valueAccessor,
+ long defaultVal) throws HyracksDataException {
+ String value = toLower(valueAccessor.apply(request, parameterName));
+ if (value == null) {
+ return defaultVal;
+ }
+ try {
+ return Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, parameterName, value);
+ }
+ }
+
+ private static <R> long parseTime(R request, String parameterName, BiFunction<R, String, String> valueAccessor,
+ long def) throws HyracksDataException {
+ String value = toLower(valueAccessor.apply(request, parameterName));
+ if (value == null) {
+ return def;
+ }
+ try {
+ return TimeUnit.NANOSECONDS.toMillis(Duration.parseDurationStringToNanos(value));
+ } catch (HyracksDataException e) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, parameterName, value);
+ }
+ }
+
+ private <R> void setFormatIfExists(R request, String acceptHeader, String parameterName,
+ BiFunction<R, String, String> valueAccessor) throws HyracksDataException {
+ Pair<OutputFormat, Boolean> formatAndHeader =
+ parseFormatIfExists(request, acceptHeader, parameterName, valueAccessor);
+ if (formatAndHeader != null) {
+ setFormat(formatAndHeader);
+ }
+ }
+
+ protected <R> Pair<OutputFormat, Boolean> parseFormatIfExists(R request, String acceptHeader, String parameterName,
+ BiFunction<R, String, String> valueAccessor) throws HyracksDataException {
+ String value = toLower(valueAccessor.apply(request, parameterName));
+ if (value == null) {
+ // if no value is provided in request parameter "format", then check "Accept" parameter in HEADER
+ // and only validate attribute val if mime and attribute name are known, e.g. application/json;lossless=?
+ if (acceptHeader != null) {
+ String[] mimeTypes = StringUtils.split(acceptHeader, ',');
+ for (int i = 0, size = mimeTypes.length; i < size; i++) {
+ Pair<OutputFormat, Boolean> formatAndHeader = fromMime(mimeTypes[i]);
+ if (formatAndHeader != null) {
+ return formatAndHeader;
+ }
+ }
+ }
+ return null;
+ }
+ // checking value in request parameter "format"
+ if (value.equals(HttpUtil.ContentType.CSV)) {
+ return Pair.of(OutputFormat.CSV, Boolean.FALSE);
+ } else if (value.equals(HttpUtil.ContentType.JSON)) {
+ return Pair.of(OutputFormat.CLEAN_JSON, Boolean.FALSE);
+ } else {
+ Pair<OutputFormat, Boolean> format = fromMime(value);
+ if (format == null) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, parameterName, value);
+ }
+ return format;
+ }
+ }
+
+ private static Pair<OutputFormat, Boolean> fromMime(String mimeType) throws HyracksDataException {
+ // find the first match, no preferences for now
+ String[] mimeSplits = StringUtils.split(mimeType, ';');
+ if (mimeSplits.length > 0) {
+ String format = mimeSplits[0].toLowerCase().trim();
+ if (format.equals(HttpUtil.ContentType.APPLICATION_JSON)) {
+ return Pair.of(hasValue(mimeSplits, Attribute.LOSSLESS.str(), booleanValues)
+ ? OutputFormat.LOSSLESS_JSON : OutputFormat.CLEAN_JSON, Boolean.FALSE);
+ } else if (format.equals(HttpUtil.ContentType.TEXT_CSV)) {
+ return Pair.of(OutputFormat.CSV,
+ hasValue(mimeSplits, Attribute.HEADER.str(), csvHeaderValues) ? Boolean.TRUE : Boolean.FALSE);
+ } else if (format.equals(HttpUtil.ContentType.APPLICATION_ADM)) {
+ return Pair.of(OutputFormat.ADM, Boolean.FALSE);
+ }
+ }
+ return null;
+ }
+
+ private static boolean hasValue(String[] mimeTypeParts, String attributeName, Map<String, Boolean> allowedValues)
+ throws HyracksDataException {
+ for (int i = 1, size = mimeTypeParts.length; i < size; i++) {
+ String[] attNameAndVal = StringUtils.split(mimeTypeParts[i], '=');
+ if (attNameAndVal.length == 2 && attNameAndVal[0].toLowerCase().trim().equals(attributeName)) {
+ Boolean value = allowedValues.get(attNameAndVal[1].toLowerCase().trim());
+ if (value == null) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, attributeName, attNameAndVal[1]);
+ }
+ return value.booleanValue();
+ }
+ }
+ return false;
+ }
+
+ private static <Req, Param> Param parseIfExists(Req request, String parameterName,
+ BiFunction<Req, String, String> valueAccessor, Param defaultVal, Function<String, Param> parseFunction)
+ throws HyracksDataException {
+ String valueInRequest = toLower(valueAccessor.apply(request, parameterName));
+ if (valueInRequest == null) {
+ return defaultVal;
+ }
+ Param resultValue = parseFunction.apply(valueInRequest);
+ if (resultValue == null) {
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, parameterName, valueInRequest);
+ }
+ return resultValue;
+ }
+
+ protected static String getParameter(JsonNode node, String parameter) {
+ final JsonNode value = node.get(parameter);
+ return value != null ? value.asText() : null;
+ }
+
+ protected static String toLower(String s) {
+ return s != null ? s.toLowerCase() : s;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 72557be..63678f8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -19,6 +19,8 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.common.exceptions.ErrorCode.ASTERIX;
+import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_JSON_VAL;
+import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_PARAM_VAL;
import static org.apache.asterix.common.exceptions.ErrorCode.NO_STATEMENT_PROVIDED;
import static org.apache.asterix.common.exceptions.ErrorCode.REJECT_BAD_CLUSTER_STATE;
import static org.apache.asterix.common.exceptions.ErrorCode.REJECT_NODE_UNREGISTERED;
@@ -33,13 +35,10 @@
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
-import java.util.function.BiFunction;
import java.util.function.Function;
import org.apache.asterix.algebra.base.ILangExtension;
@@ -65,7 +64,6 @@
import org.apache.asterix.common.api.ICodedMessage;
import org.apache.asterix.common.api.IReceptionist;
import org.apache.asterix.common.api.IRequestReference;
-import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -97,14 +95,9 @@
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.utils.HttpUtil;
import org.apache.hyracks.util.LogRedactionUtil;
-import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.JsonNode;
-
import io.netty.handler.codec.http.HttpResponseStatus;
public class QueryServiceServlet extends AbstractQueryApiServlet {
@@ -154,54 +147,6 @@
response.setStatus(HttpResponseStatus.OK);
}
- public enum Parameter {
- ARGS("args"),
- STATEMENT("statement"),
- FORMAT("format"),
- CLIENT_ID("client_context_id"),
- PRETTY("pretty"),
- MODE("mode"),
- TIMEOUT("timeout"),
- PLAN_FORMAT("plan-format"),
- MAX_RESULT_READS("max-result-reads"),
- EXPRESSION_TREE("expression-tree"),
- REWRITTEN_EXPRESSION_TREE("rewritten-expression-tree"),
- LOGICAL_PLAN("logical-plan"),
- OPTIMIZED_LOGICAL_PLAN("optimized-logical-plan"),
- PARSE_ONLY("parse-only"),
- READ_ONLY("readonly"),
- JOB("job"),
- PROFILE("profile"),
- SIGNATURE("signature"),
- MULTI_STATEMENT("multi-statement"),
- MAX_WARNINGS("max-warnings");
-
- private final String str;
-
- Parameter(String str) {
- this.str = str;
- }
-
- public String str() {
- return str;
- }
- }
-
- private enum Attribute {
- HEADER("header"),
- LOSSLESS("lossless");
-
- private final String str;
-
- Attribute(String str) {
- this.str = str;
- }
-
- public String str() {
- return str;
- }
- }
-
protected static final class RequestExecutionState {
private long execStart = -1;
private long execEnd = -1;
@@ -242,44 +187,6 @@
}
}
- private static String getParameterValue(String content, String attribute) {
- if (content == null || attribute == null) {
- return null;
- }
- int sc = content.indexOf(';');
- if (sc < 0) {
- return null;
- }
- int eq = content.indexOf('=', sc + 1);
- if (eq < 0) {
- return null;
- }
- if (content.substring(sc + 1, eq).trim().equalsIgnoreCase(attribute)) {
- return content.substring(eq + 1).trim().toLowerCase();
- }
- return null;
- }
-
- private static String toLower(String s) {
- return s != null ? s.toLowerCase() : s;
- }
-
- private static SessionConfig.OutputFormat getFormat(String format) {
- if (format != null) {
- if (format.startsWith(HttpUtil.ContentType.CSV)) {
- return SessionConfig.OutputFormat.CSV;
- }
- if (format.equals(HttpUtil.ContentType.APPLICATION_ADM)) {
- return SessionConfig.OutputFormat.ADM;
- }
- if (isJsonFormat(format)) {
- return Boolean.parseBoolean(getParameterValue(format, Attribute.LOSSLESS.str()))
- ? SessionConfig.OutputFormat.LOSSLESS_JSON : SessionConfig.OutputFormat.CLEAN_JSON;
- }
- }
- return SessionConfig.OutputFormat.CLEAN_JSON;
- }
-
private static SessionOutput createSessionOutput(PrintWriter resultWriter) {
SessionOutput.ResultDecorator resultPrefix = ResultUtil.createPreResultDecorator();
SessionOutput.ResultDecorator resultPostfix = ResultUtil.createPostResultDecorator();
@@ -288,162 +195,9 @@
return new SessionOutput(sessionConfig, resultWriter, resultPrefix, resultPostfix, null, appendStatus);
}
- protected String getOptText(JsonNode node, Parameter parameter) {
- return getOptText(node, parameter.str());
- }
-
- protected String getOptText(JsonNode node, String fieldName) {
- final JsonNode value = node.get(fieldName);
- return value != null ? value.asText() : null;
- }
-
- protected boolean getOptBoolean(JsonNode node, Parameter parameter, boolean defaultValue) {
- return getOptBoolean(node, parameter.str(), defaultValue);
- }
-
- protected boolean getOptBoolean(JsonNode node, String fieldName, boolean defaultValue) {
- final JsonNode value = node.get(fieldName);
- return value != null ? value.asBoolean() : defaultValue;
- }
-
- protected long getOptLong(JsonNode node, Parameter parameter, long defaultValue) {
- final JsonNode value = node.get(parameter.str);
- return value != null ? Integer.parseInt(value.asText()) : defaultValue;
- }
-
- protected long getOptLong(IServletRequest request, Parameter parameter, long defaultValue) {
- String value = getParameter(request, parameter);
- return value == null ? defaultValue : Integer.parseInt(value);
- }
-
- protected boolean getOptBoolean(IServletRequest request, Parameter parameter, boolean defaultValue) {
- String value = getParameter(request, parameter);
- return value == null ? defaultValue : Boolean.parseBoolean(value);
- }
-
- protected String getParameter(IServletRequest request, Parameter parameter) {
- return request.getParameter(parameter.str());
- }
-
- @FunctionalInterface
- interface CheckedFunction<I, O> {
- O apply(I requestParamValue) throws IOException;
- }
-
- private <R, P> Map<String, JsonNode> getOptStatementParameters(R request, Iterator<String> paramNameIter,
- BiFunction<R, String, P> paramValueAccessor, CheckedFunction<P, JsonNode> paramValueParser)
- throws IOException {
- Map<String, JsonNode> result = null;
- while (paramNameIter.hasNext()) {
- String paramName = paramNameIter.next();
- String stmtParamName = extractStatementParameterName(paramName);
- if (stmtParamName != null) {
- if (result == null) {
- result = new HashMap<>();
- }
- P paramValue = paramValueAccessor.apply(request, paramName);
- JsonNode stmtParamValue = paramValueParser.apply(paramValue);
- result.put(stmtParamName, stmtParamValue);
- } else if (Parameter.ARGS.str().equals(paramName)) {
- if (result == null) {
- result = new HashMap<>();
- }
- P paramValue = paramValueAccessor.apply(request, paramName);
- JsonNode stmtParamValue = paramValueParser.apply(paramValue);
- if (stmtParamValue.isArray()) {
- for (int i = 0, ln = stmtParamValue.size(); i < ln; i++) {
- result.put(String.valueOf(i + 1), stmtParamValue.get(i));
- }
- }
- }
- }
- return result;
- }
-
protected void setRequestParam(IServletRequest request, QueryServiceRequestParameters param,
Map<String, String> optionalParams) throws IOException, AlgebricksException {
- param.setHost(host(request));
- param.setPath(servletPath(request));
- String contentType = HttpUtil.getContentTypeOnly(request);
- if (HttpUtil.ContentType.APPLICATION_JSON.equals(contentType)) {
- try {
- setParamFromJSON(request, param, optionalParams);
- } catch (JsonParseException | JsonMappingException e) {
- // if the JSON parsing fails, the statement is empty and we get an empty statement error
- GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e);
- }
- } else {
- setParamFromRequest(request, param, optionalParams);
- }
- }
-
- private void setParamFromJSON(IServletRequest request, QueryServiceRequestParameters param,
- Map<String, String> optionalParameters) throws IOException {
- JsonNode jsonRequest = OBJECT_MAPPER.readTree(HttpUtil.getRequestBody(request));
- param.setStatement(getOptText(jsonRequest, Parameter.STATEMENT));
- param.setFormat(toLower(getOptText(jsonRequest, Parameter.FORMAT)));
- param.setPretty(getOptBoolean(jsonRequest, Parameter.PRETTY, false));
- param.setMode(toLower(getOptText(jsonRequest, Parameter.MODE)));
- param.setClientContextID(getOptText(jsonRequest, Parameter.CLIENT_ID));
- param.setTimeout(getOptText(jsonRequest, Parameter.TIMEOUT));
- param.setMaxResultReads(getOptText(jsonRequest, Parameter.MAX_RESULT_READS));
- param.setPlanFormat(getOptText(jsonRequest, Parameter.PLAN_FORMAT));
- param.setExpressionTree(getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE, false));
- param.setRewrittenExpressionTree(getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE, false));
- param.setLogicalPlan(getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN, false));
- param.setParseOnly(getOptBoolean(jsonRequest, Parameter.PARSE_ONLY, false));
- param.setReadOnly(getOptBoolean(jsonRequest, Parameter.READ_ONLY, false));
- param.setOptimizedLogicalPlan(getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN, false));
- param.setJob(getOptBoolean(jsonRequest, Parameter.JOB, false));
- param.setProfile(getOptBoolean(jsonRequest, Parameter.PROFILE, false));
- param.setSignature(getOptBoolean(jsonRequest, Parameter.SIGNATURE, true));
- param.setStatementParams(
- getOptStatementParameters(jsonRequest, jsonRequest.fieldNames(), JsonNode::get, v -> v));
- param.setMultiStatement(getOptBoolean(jsonRequest, Parameter.MULTI_STATEMENT, true));
- param.setMaxWarnings(
- getOptLong(jsonRequest, Parameter.MAX_WARNINGS, QueryServiceRequestParameters.DEFAULT_MAX_WARNINGS));
- setJsonOptionalParameters(jsonRequest, param, optionalParameters);
- }
-
- protected void setJsonOptionalParameters(JsonNode jsonRequest, QueryServiceRequestParameters param,
- Map<String, String> optionalParameters) {
- // allows extensions to set extra parameters
- }
-
- private void setParamFromRequest(IServletRequest request, QueryServiceRequestParameters param,
- Map<String, String> optionalParameters) throws IOException {
- param.setStatement(getParameter(request, Parameter.STATEMENT));
- param.setFormat(toLower(getParameter(request, Parameter.FORMAT)));
- param.setPretty(Boolean.parseBoolean(getParameter(request, Parameter.PRETTY)));
- param.setMode(toLower(getParameter(request, Parameter.MODE)));
- param.setClientContextID(getParameter(request, Parameter.CLIENT_ID));
- param.setTimeout(getParameter(request, Parameter.TIMEOUT));
- param.setMaxResultReads(getParameter(request, Parameter.MAX_RESULT_READS));
- param.setPlanFormat(getParameter(request, Parameter.PLAN_FORMAT));
- param.setExpressionTree(getOptBoolean(request, Parameter.EXPRESSION_TREE, false));
- param.setRewrittenExpressionTree(getOptBoolean(request, Parameter.REWRITTEN_EXPRESSION_TREE, false));
- param.setLogicalPlan(getOptBoolean(request, Parameter.LOGICAL_PLAN, false));
- param.setParseOnly(getOptBoolean(request, Parameter.PARSE_ONLY, false));
- param.setReadOnly(getOptBoolean(request, Parameter.READ_ONLY, false));
- param.setOptimizedLogicalPlan(getOptBoolean(request, Parameter.OPTIMIZED_LOGICAL_PLAN, false));
- param.setJob(getOptBoolean(request, Parameter.JOB, false));
- param.setProfile(getOptBoolean(request, Parameter.PROFILE, false));
- param.setSignature(getOptBoolean(request, Parameter.SIGNATURE, true));
- param.setMultiStatement(getOptBoolean(request, Parameter.MULTI_STATEMENT, true));
- param.setMaxWarnings(
- getOptLong(request, Parameter.MAX_WARNINGS, QueryServiceRequestParameters.DEFAULT_MAX_WARNINGS));
- try {
- param.setStatementParams(getOptStatementParameters(request, request.getParameterNames().iterator(),
- IServletRequest::getParameter, OBJECT_MAPPER::readTree));
- } catch (JsonParseException | JsonMappingException e) {
- GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e);
- }
- setOptionalParameters(request, param, optionalParameters);
- }
-
- protected void setOptionalParameters(IServletRequest request, QueryServiceRequestParameters param,
- Map<String, String> optionalParameters) {
- // allows extensions to set extra parameters
+ param.setParameters(this, request, optionalParams);
}
private void setAccessControlHeaders(IServletRequest request, IServletResponse response) throws IOException {
@@ -454,16 +208,6 @@
response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
}
- private static ResultDelivery parseResultDelivery(String mode) {
- if ("async".equals(mode)) {
- return ResultDelivery.ASYNC;
- } else if ("deferred".equals(mode)) {
- return ResultDelivery.DEFERRED;
- } else {
- return ResultDelivery.IMMEDIATE;
- }
- }
-
private static String handlePath(ResultDelivery delivery) {
switch (delivery) {
case ASYNC:
@@ -515,10 +259,9 @@
}
setRequestParam(request, param, optionalParams);
LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.userData(param.toString()));
- delivery = parseResultDelivery(param.getMode());
+ delivery = param.getMode();
setSessionConfig(sessionOutput, param, delivery);
- final ResultProperties resultProperties = param.getMaxResultReads() == null ? new ResultProperties(delivery)
- : new ResultProperties(delivery, Long.parseLong(param.getMaxResultReads()));
+ final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads());
buildResponseHeaders(requestRef, sessionOutput, param, responsePrinter, delivery);
responsePrinter.printHeaders();
validateStatement(param.getStatement());
@@ -533,7 +276,7 @@
.serializeParameterValues(param.getStatementParams());
setAccessControlHeaders(request, response);
response.setStatus(execution.getHttpStatus());
- stats.setType(param.isProfile() ? Stats.ProfileType.FULL : Stats.ProfileType.COUNTS);
+ stats.setProfileType(param.getProfileType());
executeStatement(requestRef, statementsText, sessionOutput, resultProperties, stats, param, execution,
optionalParams, statementParams, responsePrinter, warnings);
}
@@ -671,6 +414,8 @@
+ LogRedactionUtil.userData(param.toString()));
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE);
break;
+ case ASTERIX + INVALID_REQ_PARAM_VAL:
+ case ASTERIX + INVALID_REQ_JSON_VAL:
case ASTERIX + NO_STATEMENT_PROVIDED:
case HYRACKS + JOB_REQUIREMENTS_EXCEED_CAPACITY:
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
@@ -693,9 +438,8 @@
String handleUrl = getHandleUrl(param.getHost(), param.getPath(), delivery);
sessionOutput.setHandleAppender(ResultUtil.createResultHandleAppender(handleUrl));
SessionConfig sessionConfig = sessionOutput.config();
- SessionConfig.OutputFormat format = getFormat(param.getFormat());
- SessionConfig.PlanFormat planFormat = SessionConfig.PlanFormat.get(param.getPlanFormat(), param.getPlanFormat(),
- SessionConfig.PlanFormat.JSON, LOGGER);
+ SessionConfig.OutputFormat format = param.getFormat();
+ SessionConfig.PlanFormat planFormat = param.getPlanFormat();
sessionConfig.setFmt(format);
sessionConfig.setPlanFormat(planFormat);
sessionConfig.setMaxWarnings(param.getMaxWarnings());
@@ -708,8 +452,7 @@
sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.isPretty());
sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD,
format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON);
- sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, format == SessionConfig.OutputFormat.CSV
- && "present".equals(getParameterValue(param.getFormat(), Attribute.HEADER.str())));
+ sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, param.isCSVWithHeader());
}
protected void requestFailed(Throwable throwable, ResponsePrinter responsePrinter) {
@@ -721,33 +464,7 @@
return new QueryServiceRequestParameters();
}
- private static boolean isJsonFormat(String format) {
- return format.startsWith(HttpUtil.ContentType.APPLICATION_JSON)
- || format.equalsIgnoreCase(HttpUtil.ContentType.JSON);
- }
-
- public static String extractStatementParameterName(String name) {
- int ln = name.length();
- if ((ln == 2 || isStatementParameterNameRest(name, 2)) && name.charAt(0) == '$'
- && Character.isLetter(name.charAt(1))) {
- return name.substring(1);
- }
- return null;
- }
-
- private static boolean isStatementParameterNameRest(CharSequence input, int startIndex) {
- int i = startIndex;
- for (int ln = input.length(); i < ln; i++) {
- char c = input.charAt(i);
- boolean ok = c == '_' || Character.isLetterOrDigit(c);
- if (!ok) {
- return false;
- }
- }
- return i > startIndex;
- }
-
private static boolean isPrintingProfile(IStatementExecutor.Stats stats) {
- return stats.getType() == Stats.ProfileType.FULL && stats.getJobProfile() != null;
+ return stats.getProfileType() == Stats.ProfileType.FULL && stats.getJobProfile() != null;
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index e58ad06..94080da 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -19,9 +19,6 @@
package org.apache.asterix.app.message;
-import static org.apache.asterix.translator.IStatementExecutor.Stats.ProfileType.COUNTS;
-import static org.apache.asterix.translator.IStatementExecutor.Stats.ProfileType.FULL;
-
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
@@ -53,6 +50,7 @@
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.translator.IRequestParameters;
import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutor.Stats.ProfileType;
import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.asterix.translator.ResultProperties;
import org.apache.asterix.translator.SessionConfig;
@@ -87,13 +85,13 @@
private final Map<String, byte[]> statementParameters;
private final boolean multiStatement;
private final int statementCategoryRestrictionMask;
- private final boolean profile;
+ private final ProfileType profileType;
private final IRequestReference requestReference;
public ExecuteStatementRequestMessage(String requestNodeId, long requestMessageId, ILangExtension.Language lang,
String statementsText, SessionConfig sessionConfig, ResultProperties resultProperties,
String clientContextID, String handleUrl, Map<String, String> optionalParameters,
- Map<String, byte[]> statementParameters, boolean multiStatement, boolean profile,
+ Map<String, byte[]> statementParameters, boolean multiStatement, ProfileType profileType,
int statementCategoryRestrictionMask, IRequestReference requestReference) {
this.requestNodeId = requestNodeId;
this.requestMessageId = requestMessageId;
@@ -107,7 +105,7 @@
this.statementParameters = statementParameters;
this.multiStatement = multiStatement;
this.statementCategoryRestrictionMask = statementCategoryRestrictionMask;
- this.profile = profile;
+ this.profileType = profileType;
this.requestReference = requestReference;
}
@@ -147,7 +145,7 @@
IStatementExecutor translator = statementExecutorFactory.create(ccAppCtx, statements, sessionOutput,
compilationProvider, storageComponentProvider, new ResponsePrinter(sessionOutput));
final IStatementExecutor.Stats stats = new IStatementExecutor.Stats();
- stats.setType(profile ? FULL : COUNTS);
+ stats.setProfileType(profileType);
Map<String, IAObject> stmtParams = RequestParameters.deserializeParameterValues(statementParameters);
final IRequestParameters requestParameters = new RequestParameters(requestReference, statementsText, null,
resultProperties, stats, outMetadata, clientContextID, optionalParameters, stmtParams,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
index 420585a..cbc4d94 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.app.nc;
-import java.io.BufferedWriter;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -36,6 +35,7 @@
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.annotations.ThreadSafe;
+import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -191,9 +191,7 @@
if (checkpointPath.toFile().exists()) {
Files.delete(checkpointPath);
}
- try (BufferedWriter writer = Files.newBufferedWriter(checkpointPath)) {
- writer.write(checkpoint.asJson());
- }
+ FileUtil.writeAndForce(checkpointPath, checkpoint.asJson().getBytes());
// ensure it was written correctly by reading it
read(checkpointPath);
return;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/TypePrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/TypePrinter.java
index 788fbcf..ad3b8d8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/TypePrinter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/TypePrinter.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.app.result.fields;
-import static org.apache.hyracks.http.server.utils.HttpUtil.ContentType.CSV;
+import static org.apache.hyracks.http.server.utils.HttpUtil.ContentType.TEXT_CSV;
import java.io.PrintWriter;
@@ -43,8 +43,8 @@
ResultUtil.printField(pw, FIELD_NAME, HttpUtil.ContentType.APPLICATION_ADM, false);
break;
case CSV:
- String contentType =
- CSV + "; header=" + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent");
+ String contentType = TEXT_CSV + "; header="
+ + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent");
ResultUtil.printField(pw, FIELD_NAME, contentType, false);
break;
default:
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e5d0cc3..8dd8d02 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -393,7 +393,7 @@
metadataProvider.setResultAsyncMode(
resultDelivery == ResultDelivery.ASYNC || resultDelivery == ResultDelivery.DEFERRED);
metadataProvider.setMaxResultReads(maxResultReads);
- if (stats.getType() == Stats.ProfileType.FULL) {
+ if (stats.getProfileType() == Stats.ProfileType.FULL) {
this.jobFlags.add(JobFlag.PROFILE_RUNTIME);
}
handleQuery(metadataProvider, (Query) stmt, hcc, resultSet, resultDelivery, outMetadata, stats,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 8dc04eb..71ef783 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -66,7 +66,7 @@
import java.util.regex.Pattern;
import java.util.stream.Stream;
-import org.apache.asterix.api.http.server.QueryServiceServlet;
+import org.apache.asterix.api.http.server.QueryServiceRequestParameters;
import org.apache.asterix.app.external.IExternalUDFLibrarian;
import org.apache.asterix.common.api.Duration;
import org.apache.asterix.common.config.GlobalConfig;
@@ -626,13 +626,14 @@
}
public List<Parameter> constructQueryParameters(String str, OutputFormat fmt, List<Parameter> params) {
- List<Parameter> newParams = upsertParam(params, "format", ParameterTypeEnum.STRING, fmt.mimeType());
+ List<Parameter> newParams = upsertParam(params, QueryServiceRequestParameters.Parameter.FORMAT.str(),
+ ParameterTypeEnum.STRING, fmt.mimeType());
- newParams = upsertParam(newParams, QueryServiceServlet.Parameter.PLAN_FORMAT.str(), ParameterTypeEnum.STRING,
- DEFAULT_PLAN_FORMAT);
+ newParams = upsertParam(newParams, QueryServiceRequestParameters.Parameter.PLAN_FORMAT.str(),
+ ParameterTypeEnum.STRING, DEFAULT_PLAN_FORMAT);
final Optional<String> maxReadsOptional = extractMaxResultReads(str);
if (maxReadsOptional.isPresent()) {
- newParams = upsertParam(newParams, QueryServiceServlet.Parameter.MAX_RESULT_READS.str(),
+ newParams = upsertParam(newParams, QueryServiceRequestParameters.Parameter.MAX_RESULT_READS.str(),
ParameterTypeEnum.STRING, maxReadsOptional.get());
}
return newParams;
@@ -2186,7 +2187,7 @@
protected static boolean containsClientContextID(String statement) {
List<Parameter> httpParams = extractParameters(statement);
return httpParams.stream().map(Parameter::getName)
- .anyMatch(QueryServiceServlet.Parameter.CLIENT_ID.str()::equals);
+ .anyMatch(QueryServiceRequestParameters.Parameter.CLIENT_ID.str()::equals);
}
private static boolean isCancellable(String type) {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
index 7695698..619ce8c 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
@@ -33,7 +33,7 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
-import org.apache.asterix.api.http.server.QueryServiceServlet;
+import org.apache.asterix.api.http.server.QueryServiceRequestParameters;
import org.apache.asterix.app.translator.RequestParameters;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.testframework.xml.ParameterTypeEnum;
@@ -126,10 +126,10 @@
throw new IllegalArgumentException(String.valueOf(paramType));
}
- String name = QueryServiceServlet.extractStatementParameterName(paramName);
+ String name = QueryServiceRequestParameters.extractStatementParameterName(paramName);
if (name != null) {
stmtParams.put(name, paramJsonValue);
- } else if (QueryServiceServlet.Parameter.ARGS.str().equals(paramName)) {
+ } else if (QueryServiceRequestParameters.Parameter.ARGS.str().equals(paramName)) {
if (paramJsonValue.isArray()) {
for (int i = 0, ln = paramJsonValue.size(); i < ln; i++) {
stmtParams.put(String.valueOf(i + 1), paramJsonValue.get(i));
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.sqlpp
new file mode 100644
index 0000000..e693b46
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.sqlpp
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test indexnl left outer join without special groupby,
+ * : disallowing indexonly
+ * Expected Res : Success
+ * Runtime test : probe-pidx-with-join-btree-sidx1.5.query
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+ `screen-name` : string,
+ lang : string,
+ `friends-count` : bigint,
+ `statuses-count` : bigint,
+ name : string,
+ `followers-count` : bigint
+};
+
+create type test.TweetMessageType as
+ closed {
+ tweetid : bigint,
+ user : TwitterUserType,
+ `sender-location` : point,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string,
+ countA : bigint,
+ countB : bigint
+};
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create index twmSndLocIx on TweetMessages (`sender-location`) type rtree;
+
+create index msgCountAIx on TweetMessages (countA) type btree;
+
+create index msgCountBIx on TweetMessages (countB) type btree;
+
+create index msgTextIx on TweetMessages (`message-text`) type keyword;
+
+set noindexonly "false";
+
+select t1.tweetid as tweetid1, t1.countA as count1, t2.tweetid as tweetid2, t2.countB as count2
+from TweetMessages as t1 left outer join TweetMessages as t2 on t1.countA /*+ indexnl */ = t2.countB
+where t1.tweetid < 10
+order by t1.tweetid, t2.tweetid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03.sqlpp
new file mode 100644
index 0000000..24a84dc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03.sqlpp
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test indexnl left outer join without special groupby,
+ * : disallowing indexonly
+ * Expected Res : Success
+ * Runtime test : probe-pidx-with-join-btree-sidx1.4.query
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+ `screen-name` : string,
+ lang : string,
+ `friends-count` : bigint,
+ `statuses-count` : bigint,
+ name : string,
+ `followers-count` : bigint
+};
+
+create type test.TweetMessageType as
+ closed {
+ tweetid : bigint,
+ user : TwitterUserType,
+ `sender-location` : point,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string,
+ countA : bigint,
+ countB : bigint
+};
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create index twmSndLocIx on TweetMessages (`sender-location`) type rtree;
+
+create index msgCountAIx on TweetMessages (countA) type btree;
+
+create index msgCountBIx on TweetMessages (countB) type btree;
+
+create index msgTextIx on TweetMessages (`message-text`) type keyword;
+
+set noindexonly "true";
+
+select t1.tweetid as tweetid1, t1.countA as count1, t2.tweetid as tweetid2, t2.countB as count2
+from TweetMessages as t1 left outer join TweetMessages as t2 on t1.countA /*+ indexnl */ = t2.countB
+where t1.tweetid < 10
+order by t1.tweetid, t2.tweetid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
index f3dbc01..b6d1d39 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
@@ -48,17 +48,17 @@
-- HASH_PARTITION_EXCHANGE [$$238] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$285][$$207] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$285] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$294][$$207] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$294] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- UNNEST |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$289] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$298] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
- -- MICRO_PRE_CLUSTERED_GROUP_BY[$$291, $$293] |LOCAL|
+ -- MICRO_PRE_CLUSTERED_GROUP_BY[$$300, $$302] |LOCAL|
{
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
@@ -68,8 +68,8 @@
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$289(ASC), $$291(ASC), $$293(ASC)] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$289] |PARTITIONED|
+ -- STABLE_SORT [$$298(ASC), $$300(ASC), $$302(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$298] |PARTITIONED|
-- UNION_ALL |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -86,45 +86,40 @@
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- RTREE_SEARCH |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$306(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- RTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- NESTED_LOOP |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$245(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$245(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
@@ -137,45 +132,40 @@
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- RTREE_SEARCH |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$306(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- RTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- NESTED_LOOP |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$245(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$245(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -194,4 +184,4 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- DATASOURCE_SCAN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.plan
new file mode 100644
index 0000000..c6142aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.plan
@@ -0,0 +1,49 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$63(ASC), $$38(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$63(ASC), $$38(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$63(ASC), $$38(DESC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$63] |PARTITIONED|
+ -- UNION_ALL |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03.plan
new file mode 100644
index 0000000..ce24784
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03.plan
@@ -0,0 +1,33 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$37(ASC), $$38(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$37(ASC), $$38(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$37(ASC), $$38(DESC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$37] |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$52(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/profiled.xml b/asterixdb/asterix-app/src/test/resources/runtimets/profiled.xml
index 496927e..f3209a1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/profiled.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/profiled.xml
@@ -22,9 +22,21 @@
<test-group name="profile">
<test-case FilePath="profile">
<compilation-unit name="full-scan">
- <parameter name="profile" value="true" />
+ <parameter name="profile" value="timings" type="string"/>
<output-dir compare="Text">full-scan</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="profile">
+ <compilation-unit name="full-scan-2">
+ <parameter name="profile" value="counts" type="string"/>
+ <output-dir compare="Text">full-scan-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="profile">
+ <compilation-unit name="full-scan-3">
+ <parameter name="profile" value="off" type="string"/>
+ <output-dir compare="Text">full-scan-3</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-01/format-param-in-accept-01.1.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-01/format-param-in-accept-01.1.post.http
new file mode 100644
index 0000000..cf0c800
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-01/format-param-in-accept-01.1.post.http
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description: testing specifying application/json as output format in Accept of HEADER
+ */
+/query/service
+--body={"statement": "set `import-private-functions` `true`; `unordered-list-constructor`(\"foo\", \"bar\");"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-02/format-param-in-accept-02.1.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-02/format-param-in-accept-02.1.post.http
new file mode 100644
index 0000000..cf0c800
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-02/format-param-in-accept-02.1.post.http
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description: testing specifying application/json as output format in Accept of HEADER
+ */
+/query/service
+--body={"statement": "set `import-private-functions` `true`; `unordered-list-constructor`(\"foo\", \"bar\");"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-03/format-param-in-accept-03.1.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-03/format-param-in-accept-03.1.post.http
new file mode 100644
index 0000000..bb7055a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-03/format-param-in-accept-03.1.post.http
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description: testing specifying application/json;lossless=true as output format in Accept of HEADER
+ */
+/query/service
+--body={"statement": "set `import-private-functions` `true`; `unordered-list-constructor`(\"foo\", \"bar\");"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-04/format-param-in-accept-04.1.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-04/format-param-in-accept-04.1.post.http
new file mode 100644
index 0000000..c68fabe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-04/format-param-in-accept-04.1.post.http
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description: testing specifying application/x-adm as output format in Accept of HEADER and json in "format" param
+ * Expected : json format is picked
+ */
+/query/service
+--body={"statement": "set `import-private-functions` `true`; `unordered-list-constructor`(\"foo\", \"bar\");", "format": "json"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-05/format-param-in-accept-05.1.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-05/format-param-in-accept-05.1.post.http
new file mode 100644
index 0000000..720b5f2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/format-param-in-accept-05/format-param-in-accept-05.1.post.http
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description: testing specifying unknown mime type, application/x-ast
+ * Expected : json is defaulted since no "format" parameter is specified and format in Accept is unkownn
+ */
+/query/service
+--body={"statement": "set `import-private-functions` `true`; `unordered-list-constructor`(\"foo\", \"bar\");"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.post.http
new file mode 100644
index 0000000..8c5a4d6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.post.http
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// statuscode 400
+/query/service
+--body={"statement": "from [1, 2] as v select v;", "format": "foo"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.001.query.sqlpp
new file mode 100644
index 0000000..69e67be
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.001.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: validating that request parameters have proper values
+ */
+
+// requesttype=application/json
+// param format:string=foo
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.002.query.sqlpp
new file mode 100644
index 0000000..0ab2195
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.002.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param pretty:string=bar
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.003.query.sqlpp
new file mode 100644
index 0000000..24267b1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.003.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param plan-format:string=blah
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.004.query.sqlpp
new file mode 100644
index 0000000..68ba956
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.004.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param max-result-reads:string=foo
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.005.query.sqlpp
new file mode 100644
index 0000000..2c8b15e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.005.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param max-result-reads:string=9999999999999999999999999999999999999999
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.006.query.sqlpp
new file mode 100644
index 0000000..f7adc7a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.006.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param max-warnings:string=baz
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.007.query.sqlpp
new file mode 100644
index 0000000..6433ae6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.007.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param max-warnings:string=1.5
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.008.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.008.query.sqlpp
new file mode 100644
index 0000000..46cd166
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.008.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param mode:string=asyn
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.009.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.009.query.sqlpp
new file mode 100644
index 0000000..29ef01b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.009.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param timeout:string=12
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.010.query.sqlpp
new file mode 100644
index 0000000..e56f9a2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.010.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param args:json=12
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.011.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.011.query.sqlpp
new file mode 100644
index 0000000..073a264
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.011.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param args:json=[12,,1]
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.012.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.012.query.sqlpp
new file mode 100644
index 0000000..1bf1146
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.012.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param $var:json={"a":}
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.013.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.013.query.sqlpp
new file mode 100644
index 0000000..837ee28
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.013.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param format:string=foo
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.014.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.014.query.sqlpp
new file mode 100644
index 0000000..25b8eb8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.014.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param pretty:string=bar
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.015.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.015.query.sqlpp
new file mode 100644
index 0000000..bdda7c4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.015.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param plan-format:string=blah
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.016.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.016.query.sqlpp
new file mode 100644
index 0000000..eedad6d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.016.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param max-result-reads:string=foo
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.017.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.017.query.sqlpp
new file mode 100644
index 0000000..fc0c48d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.017.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param max-warnings:string=baz
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.018.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.018.query.sqlpp
new file mode 100644
index 0000000..cbcf381
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.018.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param mode:string=asyn
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.019.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.019.query.sqlpp
new file mode 100644
index 0000000..d682463
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.019.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param args:json=12
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.020.query.sqlpp
new file mode 100644
index 0000000..0f58953
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.020.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param args:json=[12,,1]
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.021.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.021.query.sqlpp
new file mode 100644
index 0000000..e37d945
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.021.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param $var:json={"a":}
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.022.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.022.query.sqlpp
new file mode 100644
index 0000000..0665e09
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.022.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param profile:string=true
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.023.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.023.query.sqlpp
new file mode 100644
index 0000000..5767812
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.023.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param profile:string=true
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.024.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.024.query.sqlpp
new file mode 100644
index 0000000..e99ec6c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.024.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param profile:string=foo
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.025.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.025.query.sqlpp
new file mode 100644
index 0000000..2ca9df4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param-validation/request-param-validation.025.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/x-www-form-urlencoded
+// param profile:string=foo
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.001.query.sqlpp
new file mode 100644
index 0000000..f22230e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.001.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=json
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.002.query.sqlpp
new file mode 100644
index 0000000..b574cc9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.002.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=csv
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.003.query.sqlpp
new file mode 100644
index 0000000..ba37336
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.003.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=application/json
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.004.query.sqlpp
new file mode 100644
index 0000000..b5d0d64
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.004.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=text/csv
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.005.query.sqlpp
new file mode 100644
index 0000000..9d3b964
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.005.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=application/x-adm
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.006.query.sqlpp
new file mode 100644
index 0000000..d667f48
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.006.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=text/csv;header=absent
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.007.query.sqlpp
new file mode 100644
index 0000000..cba7638
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.007.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=application/json;lossless=true
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.008.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.008.query.sqlpp
new file mode 100644
index 0000000..7b6c450
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.008.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=application/json;lossless=false
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.009.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.009.query.sqlpp
new file mode 100644
index 0000000..d483eb3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.009.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param mode:string=immediate
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.query.sqlpp
new file mode 100644
index 0000000..e1a91bc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.query.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test indexnl left outer join without special groupby,
+ * : disallowing indexonly
+ * Expected Res : Success
+ */
+
+use test;
+
+set noindexonly "true";
+
+select t1.tweetid as tweetid1, t1.countA as count1, t2.tweetid as tweetid2, t2.countB as count2
+from TweetMessages as t1 left outer join TweetMessages as t2 on t1.countA /*+ indexnl */ = t2.countB
+where t1.tweetid < 10
+order by t1.tweetid, t2.tweetid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.query.sqlpp
new file mode 100644
index 0000000..6596c92
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.query.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test indexnl left outer join without special groupby,
+ * : allowing indexonly
+ * Expected Res : Success
+ */
+
+use test;
+
+set noindexonly "false";
+
+select t1.tweetid as tweetid1, t1.countA as count1, t2.tweetid as tweetid2, t2.countB as count2
+from TweetMessages as t1 left outer join TweetMessages as t2 on t1.countA /*+ indexnl */ = t2.countB
+where t1.tweetid < 10
+order by t1.tweetid, t2.tweetid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissing/ifmissing.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissing/ifmissing.1.query.sqlpp
index 0abb997..f0c0001 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissing/ifmissing.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissing/ifmissing.1.query.sqlpp
@@ -36,5 +36,6 @@
{ "c": [ 2 ] }
)
select v as b
- )
+ ),
+ "k": if_missing(missing, ["a"], "abc", {"id":3})
};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissingornull/ifmissingornull.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissingornull/ifmissingornull.1.query.sqlpp
index 22a8acd..735cfa9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissingornull/ifmissingornull.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifmissingornull/ifmissingornull.1.query.sqlpp
@@ -39,5 +39,6 @@
{ "c": [ 2 ] }
)
select v as b
- )
+ ),
+ "k": if_missing_or_null(null, missing, ["a"], "abc", {"id":3})
};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifnull/ifnull.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifnull/ifnull.1.query.sqlpp
index 0121cd8..6ae39e5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifnull/ifnull.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/ifnull/ifnull.1.query.sqlpp
@@ -37,5 +37,6 @@
{ "c": [ 2 ] }
)
select v as b
- )
+ ),
+ "k": if_null(null,["a"],"abc", {"id":3})
};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.1.ddl.sqlpp
new file mode 100644
index 0000000..c5cab5d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.1.ddl.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing that "counts" is supplied in request parameter "profile".
+ * Expected Res : Success with expected result not having "profile" field.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type test.AddressType as
+{
+ number : bigint,
+ street : string,
+ city : string
+};
+
+create type test.CustomerType as
+ closed {
+ cid : bigint,
+ name : string,
+ age : bigint?,
+ address : AddressType?,
+ lastorder : {
+ oid : bigint,
+ total : float
+ }
+};
+
+create dataset Customers(CustomerType) primary key cid;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.2.update.sqlpp
new file mode 100644
index 0000000..957c234
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.2.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset Customers using localfs
+ ((`path`=`asterix_nc1://data/custord-tiny/customer-tiny-neg.adm`),
+ (`format`=`adm`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.3.profile.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.3.profile.sqlpp
new file mode 100644
index 0000000..070196a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.3.profile.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select count(*) from Customers;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.4.ddl.sqlpp
new file mode 100644
index 0000000..f12a2b7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-2/full-scan-2.4.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.1.ddl.sqlpp
new file mode 100644
index 0000000..da6580a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.1.ddl.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing that "off" is supplied in request parameter "profile".
+ * Expected Res : Success with expected result not having "profile" field.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type test.AddressType as
+{
+ number : bigint,
+ street : string,
+ city : string
+};
+
+create type test.CustomerType as
+ closed {
+ cid : bigint,
+ name : string,
+ age : bigint?,
+ address : AddressType?,
+ lastorder : {
+ oid : bigint,
+ total : float
+ }
+};
+
+create dataset Customers(CustomerType) primary key cid;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.2.update.sqlpp
new file mode 100644
index 0000000..957c234
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.2.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset Customers using localfs
+ ((`path`=`asterix_nc1://data/custord-tiny/customer-tiny-neg.adm`),
+ (`format`=`adm`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.3.profile.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.3.profile.sqlpp
new file mode 100644
index 0000000..070196a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.3.profile.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select count(*) from Customers;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.4.ddl.sqlpp
new file mode 100644
index 0000000..f12a2b7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/profile/full-scan-3/full-scan-3.4.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-01/format-param-in-accept-01.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-01/format-param-in-accept-01.1.regexjson
new file mode 100644
index 0000000..ad8ba0a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-01/format-param-in-accept-01.1.regexjson
@@ -0,0 +1,9 @@
+{
+ "requestID": "R{[a-zA-Z0-9-]+}",
+ "signature": {"*": "*"},
+ "type": "application/x-adm",
+ "results": [ "{{ \"foo\", \"bar\" }}\n" ],
+ "plans":{},
+ "status": "success",
+ "metrics": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-02/format-param-in-accept-02.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-02/format-param-in-accept-02.1.regexjson
new file mode 100644
index 0000000..7443917
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-02/format-param-in-accept-02.1.regexjson
@@ -0,0 +1,8 @@
+{
+ "requestID": "R{[a-zA-Z0-9-]+}",
+ "signature": {"*": "*"},
+ "results": [["foo","bar"]],
+ "plans":{},
+ "status": "success",
+ "metrics": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-03/format-param-in-accept-03.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-03/format-param-in-accept-03.1.regexjson
new file mode 100644
index 0000000..e48a4e1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-03/format-param-in-accept-03.1.regexjson
@@ -0,0 +1,8 @@
+{
+ "requestID": "R{[a-zA-Z0-9-]+}",
+ "signature": {"*": "*"},
+ "results": [{"unorderedlist":["foo","bar"]}],
+ "plans":{},
+ "status": "success",
+ "metrics": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-04/format-param-in-accept-04.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-04/format-param-in-accept-04.1.regexjson
new file mode 100644
index 0000000..7443917
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-04/format-param-in-accept-04.1.regexjson
@@ -0,0 +1,8 @@
+{
+ "requestID": "R{[a-zA-Z0-9-]+}",
+ "signature": {"*": "*"},
+ "results": [["foo","bar"]],
+ "plans":{},
+ "status": "success",
+ "metrics": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-05/format-param-in-accept-05.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-05/format-param-in-accept-05.1.regexjson
new file mode 100644
index 0000000..7443917
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/format-param-in-accept-05/format-param-in-accept-05.1.regexjson
@@ -0,0 +1,8 @@
+{
+ "requestID": "R{[a-zA-Z0-9-]+}",
+ "signature": {"*": "*"},
+ "results": [["foo","bar"]],
+ "plans":{},
+ "status": "success",
+ "metrics": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.regexjson
new file mode 100644
index 0000000..f444fa1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param-validation-400-BAD/request-param-validation-400-BAD.01.regexjson
@@ -0,0 +1,14 @@
+{
+ "errors": [{
+ "code": 1, "msg": "ASX0047: Invalid value for parameter \"format\": foo" }
+ ],
+ "status": "fatal",
+ "metrics": {
+ "elapsedTime": "R{.*}",
+ "executionTime": "R{.*}",
+ "resultCount": 0,
+ "resultSize": 0,
+ "processedObjects": 0,
+ "errorCount": 1
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.001.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.001.json
new file mode 100644
index 0000000..83ce118
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.001.json
@@ -0,0 +1 @@
+{"v":1}{"v":2}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.002.csv b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.002.csv
new file mode 100644
index 0000000..7a754f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.002.csv
@@ -0,0 +1,2 @@
+1
+2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.003.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.003.json
new file mode 100644
index 0000000..83ce118
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.003.json
@@ -0,0 +1 @@
+{"v":1}{"v":2}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.004.csv b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.004.csv
new file mode 100644
index 0000000..7a754f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.004.csv
@@ -0,0 +1,2 @@
+1
+2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.005.adm
new file mode 100644
index 0000000..cb88308
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.005.adm
@@ -0,0 +1,2 @@
+{ "v": 1 }
+{ "v": 2 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.006.csv b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.006.csv
new file mode 100644
index 0000000..7a754f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.006.csv
@@ -0,0 +1,2 @@
+1
+2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.007.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.007.json
new file mode 100644
index 0000000..32c4637
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.007.json
@@ -0,0 +1 @@
+{"v":{"int64":1}}{"v":{"int64":2}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.008.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.008.json
new file mode 100644
index 0000000..83ce118
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.008.json
@@ -0,0 +1 @@
+{"v":1}{"v":2}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.009.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.009.adm
new file mode 100644
index 0000000..cb88308
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.009.adm
@@ -0,0 +1,2 @@
+{ "v": 1 }
+{ "v": 2 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.3.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.adm
new file mode 100644
index 0000000..75fa339
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.adm
@@ -0,0 +1,11 @@
+{ "tweetid1": 1, "count1": 1 }
+{ "tweetid1": 2, "count1": 2, "tweetid2": 60, "count2": 2 }
+{ "tweetid1": 3, "count1": 3, "tweetid2": 105, "count2": 3 }
+{ "tweetid1": 3, "count1": 3, "tweetid2": 206, "count2": 3 }
+{ "tweetid1": 4, "count1": 4 }
+{ "tweetid1": 5, "count1": 5, "tweetid2": 138, "count2": 5 }
+{ "tweetid1": 5, "count1": 5, "tweetid2": 175, "count2": 5 }
+{ "tweetid1": 6, "count1": 6, "tweetid2": 148, "count2": 6 }
+{ "tweetid1": 7, "count1": 7, "tweetid2": 125, "count2": 7 }
+{ "tweetid1": 8, "count1": 8 }
+{ "tweetid1": 9, "count1": 9, "tweetid2": 141, "count2": 9 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.adm
new file mode 100644
index 0000000..75fa339
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.adm
@@ -0,0 +1,11 @@
+{ "tweetid1": 1, "count1": 1 }
+{ "tweetid1": 2, "count1": 2, "tweetid2": 60, "count2": 2 }
+{ "tweetid1": 3, "count1": 3, "tweetid2": 105, "count2": 3 }
+{ "tweetid1": 3, "count1": 3, "tweetid2": 206, "count2": 3 }
+{ "tweetid1": 4, "count1": 4 }
+{ "tweetid1": 5, "count1": 5, "tweetid2": 138, "count2": 5 }
+{ "tweetid1": 5, "count1": 5, "tweetid2": 175, "count2": 5 }
+{ "tweetid1": 6, "count1": 6, "tweetid2": 148, "count2": 6 }
+{ "tweetid1": 7, "count1": 7, "tweetid2": 125, "count2": 7 }
+{ "tweetid1": 8, "count1": 8 }
+{ "tweetid1": 9, "count1": 9, "tweetid2": 141, "count2": 9 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissing/ifmissing.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissing/ifmissing.1.adm
index 14620a1..f416057 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissing/ifmissing.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissing/ifmissing.1.adm
@@ -1 +1 @@
-{ "a": true, "b": true, "c": true, "d": true, "e": true, "f": true, "g": true, "i": true, "j": [ { "b": { "c": [ 2 ] } } ] }
\ No newline at end of file
+{ "a": true, "b": true, "c": true, "d": true, "e": true, "f": true, "g": true, "i": true, "j": [ { "b": { "c": [ 2 ] } } ], "k": [ "a" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissingornull/ifmissingornull.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissingornull/ifmissingornull.1.adm
index eff2651..ff4b01f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissingornull/ifmissingornull.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifmissingornull/ifmissingornull.1.adm
@@ -1 +1 @@
-{ "a": true, "b": true, "c": true, "d": true, "e": true, "f": true, "g": true, "h": true, "i": true, "j": [ { "b": { "c": [ 2 ] } } ] }
\ No newline at end of file
+{ "a": true, "b": true, "c": true, "d": true, "e": true, "f": true, "g": true, "h": true, "i": true, "j": [ { "b": { "c": [ 2 ] } } ], "k": [ "a" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifnull/ifnull.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifnull/ifnull.1.adm
index eff2651..ff4b01f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifnull/ifnull.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/ifnull/ifnull.1.adm
@@ -1 +1 @@
-{ "a": true, "b": true, "c": true, "d": true, "e": true, "f": true, "g": true, "h": true, "i": true, "j": [ { "b": { "c": [ 2 ] } } ] }
\ No newline at end of file
+{ "a": true, "b": true, "c": true, "d": true, "e": true, "f": true, "g": true, "h": true, "i": true, "j": [ { "b": { "c": [ 2 ] } } ], "k": [ "a" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan-2/full-scan-2.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan-2/full-scan-2.3.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan-2/full-scan-2.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan-3/full-scan-3.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan-3/full-scan-3.3.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/profile/full-scan-3/full-scan-3.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.ast
new file mode 100644
index 0000000..4e1ebd6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.4.ast
@@ -0,0 +1,68 @@
+DataverseUse test
+Set noindexonly=true
+Query:
+SELECT [
+FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=tweetid
+]
+tweetid1
+FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=countA
+]
+count1
+FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=tweetid
+]
+tweetid2
+FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=countB
+]
+count2
+]
+FROM [ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.TweetMessages]
+ ]
+ AS Variable [ Name=$t1 ]
+ LEFTOUTER JOIN
+ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.TweetMessages]
+ ]
+ AS Variable [ Name=$t2 ]
+ ON
+ OperatorExpr [
+ FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=countA
+ ]
+ =
+ FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=countB
+ ]
+ ]
+]
+Where
+ OperatorExpr [
+ FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=tweetid
+ ]
+ <
+ LiteralExpr [LONG] [10]
+ ]
+Orderby
+ FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=tweetid
+ ]
+ ASC
+ FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=tweetid
+ ]
+ ASC
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.ast
new file mode 100644
index 0000000..965536d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.5.ast
@@ -0,0 +1,68 @@
+DataverseUse test
+Set noindexonly=false
+Query:
+SELECT [
+FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=tweetid
+]
+tweetid1
+FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=countA
+]
+count1
+FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=tweetid
+]
+tweetid2
+FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=countB
+]
+count2
+]
+FROM [ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.TweetMessages]
+ ]
+ AS Variable [ Name=$t1 ]
+ LEFTOUTER JOIN
+ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.TweetMessages]
+ ]
+ AS Variable [ Name=$t2 ]
+ ON
+ OperatorExpr [
+ FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=countA
+ ]
+ =
+ FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=countB
+ ]
+ ]
+]
+Where
+ OperatorExpr [
+ FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=tweetid
+ ]
+ <
+ LiteralExpr [LONG] [10]
+ ]
+Orderby
+ FieldAccessor [
+ Variable [ Name=$t1 ]
+ Field=tweetid
+ ]
+ ASC
+ FieldAccessor [
+ Variable [ Name=$t2 ]
+ Field=tweetid
+ ]
+ ASC
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index f30b965..b2b6bd8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -42,6 +42,72 @@
<expected-error>ASX0044: DELETE statement is prohibited by this request</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="request-param-validation">
+ <output-dir compare="Text">request-param-validation</output-dir>
+ <source-location>false</source-location>
+ <expected-error>Invalid value for parameter "format": foo</expected-error>
+ <expected-error>Invalid value for parameter "pretty": bar</expected-error>
+ <expected-error>Invalid value for parameter "plan-format": blah</expected-error>
+ <expected-error>Invalid value for parameter "max-result-reads": foo</expected-error>
+ <expected-error>Invalid value for parameter "max-result-reads": 9999999999999999999999999999999999999999</expected-error>
+ <expected-error>Invalid value for parameter "max-warnings": baz</expected-error>
+ <expected-error>Invalid value for parameter "max-warnings": 1.5</expected-error>
+ <expected-error>Invalid value for parameter "mode": asyn</expected-error>
+ <expected-error>Invalid value for parameter "timeout": 12</expected-error>
+ <expected-error>Invalid value for parameter "args": 12</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Invalid value for parameter "format": foo</expected-error>
+ <expected-error>Invalid value for parameter "pretty": bar</expected-error>
+ <expected-error>Invalid value for parameter "plan-format": blah</expected-error>
+ <expected-error>Invalid value for parameter "max-result-reads": foo</expected-error>
+ <expected-error>Invalid value for parameter "max-warnings": baz</expected-error>
+ <expected-error>Invalid value for parameter "mode": asyn</expected-error>
+ <expected-error>Invalid value for parameter "args": 12</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Unable to process JSON content in request</expected-error>
+ <expected-error>Invalid value for parameter "profile": true</expected-error>
+ <expected-error>Invalid value for parameter "profile": true</expected-error>
+ <expected-error>Invalid value for parameter "profile": foo</expected-error>
+ <expected-error>Invalid value for parameter "profile": foo</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="request-param-validation-400-BAD">
+ <output-dir compare="Text">request-param-validation-400-BAD</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="request-param">
+ <output-dir compare="Text">request-param</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-01">
+ <output-dir compare="Text">format-param-in-accept-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-02">
+ <output-dir compare="Clean-JSON">format-param-in-accept-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-03">
+ <output-dir compare="Lossless-JSON">format-param-in-accept-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-04">
+ <output-dir compare="Text">format-param-in-accept-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="format-param-in-accept-05">
+ <output-dir compare="AST">format-param-in-accept-05</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="flwor">
<test-case FilePath="flwor">
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index e718ca5..da1c4e7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -79,6 +79,8 @@
public static final int PROHIBITED_STATEMENT_CATEGORY = 44;
public static final int INTEGER_VALUE_EXPECTED_FUNCTION = 45;
public static final int INVALID_LIKE_PATTERN = 46;
+ public static final int INVALID_REQ_PARAM_VAL = 47;
+ public static final int INVALID_REQ_JSON_VAL = 48;
public static final int UNSUPPORTED_JRE = 100;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index f8d7a89..457206d 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -81,6 +81,8 @@
44 = %1$s statement is prohibited by this request
45 = Invalid value: function %1$s expects its %2$s input parameter to be an integer value, got %3$s
46 = Invalid pattern \"%1$s\" for LIKE
+47 = Invalid value for parameter \"%1$s\": %2$s
+48 = Unable to process JSON content in request
100 = Unsupported JRE: %1$s
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index f080728..1397b1a 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1026,8 +1026,8 @@
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "percent_rank", 0);
public static final FunctionIdentifier PERCENT_RANK_IMPL =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "percent-rank-impl", FunctionIdentifier.VARARGS);
- public static final FunctionIdentifier WIN_PARTITION_LENGTH =
- new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "win_partition_length", 0);
+ public static final FunctionIdentifier WIN_MARK_FIRST_MISSING_IMPL = new FunctionIdentifier(
+ FunctionConstants.ASTERIX_NS, "win-mark-first-missing-impl", FunctionIdentifier.VARARGS);
public static final FunctionIdentifier WIN_PARTITION_LENGTH_IMPL =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "win-partition-length-impl", 0);
@@ -2064,7 +2064,7 @@
addFunction(ROW_NUMBER_IMPL, AInt64TypeComputer.INSTANCE, false);
addFunction(PERCENT_RANK, ADoubleTypeComputer.INSTANCE, false);
addFunction(PERCENT_RANK_IMPL, ADoubleTypeComputer.INSTANCE, false);
- addPrivateFunction(WIN_PARTITION_LENGTH, AInt64TypeComputer.INSTANCE, false);
+ addPrivateFunction(WIN_MARK_FIRST_MISSING_IMPL, ABooleanTypeComputer.INSTANCE, false);
addPrivateFunction(WIN_PARTITION_LENGTH_IMPL, AInt64TypeComputer.INSTANCE, false);
// Similarity functions
@@ -2991,7 +2991,8 @@
addWindowFunction(RANK, RANK_IMPL, NO_FRAME_CLAUSE, INJECT_ORDER_ARGS);
addWindowFunction(RATIO_TO_REPORT, RATIO_TO_REPORT_IMPL, HAS_LIST_ARG);
addWindowFunction(ROW_NUMBER, ROW_NUMBER_IMPL, NO_FRAME_CLAUSE);
- addWindowFunction(WIN_PARTITION_LENGTH, WIN_PARTITION_LENGTH_IMPL, NO_FRAME_CLAUSE, MATERIALIZE_PARTITION);
+ addWindowFunction(null, WIN_MARK_FIRST_MISSING_IMPL, NO_FRAME_CLAUSE, INJECT_ORDER_ARGS);
+ addWindowFunction(null, WIN_PARTITION_LENGTH_IMPL, NO_FRAME_CLAUSE, MATERIALIZE_PARTITION);
}
static {
@@ -3199,9 +3200,10 @@
public static void addWindowFunction(FunctionIdentifier sqlfi, FunctionIdentifier winfi,
WindowFunctionProperty... properties) {
- IFunctionInfo sqlinfo = getAsterixFunctionInfo(sqlfi);
IFunctionInfo wininfo = getAsterixFunctionInfo(winfi);
- sqlToWindowFunctions.put(sqlinfo, wininfo);
+ if (sqlfi != null) {
+ sqlToWindowFunctions.put(getAsterixFunctionInfo(sqlfi), wininfo);
+ }
windowFunctions.add(wininfo);
registerFunctionProperties(wininfo, WindowFunctionProperty.class, properties);
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ACastVisitor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ACastVisitor.java
index 04ee28b..993edee 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ACastVisitor.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ACastVisitor.java
@@ -23,6 +23,8 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.om.pointables.AFlatValuePointable;
import org.apache.asterix.om.pointables.AListVisitablePointable;
import org.apache.asterix.om.pointables.ARecordVisitablePointable;
@@ -37,6 +39,7 @@
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
/**
@@ -57,13 +60,19 @@
private final ArrayBackedValueStorage castBuffer = new ArrayBackedValueStorage();
private final boolean strictDemote;
+ private final SourceLocation sourceLoc;
public ACastVisitor() {
- this(true);
+ this(true, null);
}
- public ACastVisitor(boolean strictDemote) {
+ public ACastVisitor(SourceLocation sourceLoc) {
+ this(true, sourceLoc);
+ }
+
+ public ACastVisitor(boolean strictDemote, SourceLocation sourceLoc) {
this.strictDemote = strictDemote;
+ this.sourceLoc = sourceLoc;
}
@Override
@@ -74,11 +83,23 @@
caster = new AListCaster();
laccessorToCaster.put(accessor, caster);
}
- if (arg.second.getTypeTag().equals(ATypeTag.ANY)) {
- arg.second = accessor.ordered() ? DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE
- : DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE;
+
+ AbstractCollectionType resultType;
+ switch (arg.second.getTypeTag()) {
+ case ANY:
+ resultType = accessor.ordered() ? DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE
+ : DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE;
+ break;
+ case ARRAY:
+ case MULTISET:
+ resultType = (AbstractCollectionType) arg.second;
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_CONVERT, sourceLoc,
+ accessor.ordered() ? ATypeTag.ARRAY : ATypeTag.MULTISET, arg.second.getTypeTag());
}
- caster.castList(accessor, arg.first, (AbstractCollectionType) arg.second, this);
+
+ caster.castList(accessor, arg.first, resultType, this);
return null;
}
@@ -90,10 +111,20 @@
caster = new ARecordCaster();
raccessorToCaster.put(accessor, caster);
}
- if (arg.second.getTypeTag().equals(ATypeTag.ANY)) {
- arg.second = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+
+ ARecordType resultType;
+ switch (arg.second.getTypeTag()) {
+ case ANY:
+ resultType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ break;
+ case OBJECT:
+ resultType = (ARecordType) arg.second;
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_CONVERT, sourceLoc, ATypeTag.OBJECT,
+ arg.second.getTypeTag());
}
- ARecordType resultType = (ARecordType) arg.second;
+
caster.castRecord(accessor, arg.first, resultType, this);
return null;
}
@@ -123,11 +154,11 @@
ATypeHierarchy.convertNumericTypeByteArray(accessor.getByteArray(), accessor.getStartOffset(),
accessor.getLength(), reqTypeTag, castBuffer.getDataOutput(), strictDemote);
arg.first.set(castBuffer);
- } catch (IOException e1) {
- throw new HyracksDataException(
- "Type mismatch: cannot cast the " + inputTypeTag + " type to the " + reqTypeTag + " type.");
+ } catch (HyracksDataException e) {
+ throw e;
+ } catch (IOException e) {
+ throw new RuntimeDataException(ErrorCode.TYPE_CONVERT, sourceLoc, inputTypeTag, reqTypeTag);
}
-
}
return null;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/TypeCastUtils.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/TypeCastUtils.java
index f61518c..50c919b 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/TypeCastUtils.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/TypeCastUtils.java
@@ -35,17 +35,25 @@
public static boolean setRequiredAndInputTypes(AbstractFunctionCallExpression expr, IAType requiredType,
IAType inputType) throws CompilationException {
+ return setRequiredAndInputTypes(expr, requiredType, inputType, true);
+ }
+
+ public static boolean setRequiredAndInputTypes(AbstractFunctionCallExpression expr, IAType requiredType,
+ IAType inputType, boolean failIfTypeMismatch) throws CompilationException {
boolean changed = false;
Object[] opaqueParameters = expr.getOpaqueParameters();
if (opaqueParameters == null) {
opaqueParameters = new Object[2];
opaqueParameters[0] = requiredType;
opaqueParameters[1] = inputType;
- ATypeTag requiredTypeTag = requiredType.getTypeTag();
- ATypeTag actualTypeTag = TypeComputeUtils.getActualType(inputType).getTypeTag();
- if (!ATypeHierarchy.isCompatible(requiredTypeTag, actualTypeTag)) {
- FunctionIdentifier funcId = expr.getFunctionIdentifier();
- throw new IncompatibleTypeException(expr.getSourceLocation(), funcId, actualTypeTag, requiredTypeTag);
+ if (failIfTypeMismatch) {
+ ATypeTag requiredTypeTag = requiredType.getTypeTag();
+ ATypeTag actualTypeTag = TypeComputeUtils.getActualType(inputType).getTypeTag();
+ if (!ATypeHierarchy.isCompatible(requiredTypeTag, actualTypeTag)) {
+ FunctionIdentifier funcId = expr.getFunctionIdentifier();
+ throw new IncompatibleTypeException(expr.getSourceLocation(), funcId, actualTypeTag,
+ requiredTypeTag);
+ }
}
expr.setOpaqueParameters(opaqueParameters);
changed = true;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeDescriptor.java
index 8080e16..c27423b 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeDescriptor.java
@@ -88,7 +88,8 @@
@Override
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
- return new CastTypeEvaluator(reqType, inputType, recordEvalFactory.createScalarEvaluator(ctx));
+ IScalarEvaluator argEval = recordEvalFactory.createScalarEvaluator(ctx);
+ return new CastTypeEvaluator(reqType, inputType, argEval, sourceLoc);
}
};
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java
index e8c2f6e..210149c 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
@@ -36,6 +37,7 @@
public class CastTypeEvaluator implements IScalarEvaluator {
private IScalarEvaluator argEvaluator;
+ protected final SourceLocation sourceLoc;
private final IPointable argPointable = new VoidPointable();
private final PointableAllocator allocator = new PointableAllocator();
private IVisitablePointable inputPointable;
@@ -44,10 +46,22 @@
private final Triple<IVisitablePointable, IAType, Boolean> arg = new Triple<>(null, null, null);
public CastTypeEvaluator() {
+ this(null);
+ // reset() should be called after using this constructor before calling any method
+ }
+
+ public CastTypeEvaluator(SourceLocation sourceLoc) {
+ this.sourceLoc = sourceLoc;
// reset() should be called after using this constructor before calling any method
}
public CastTypeEvaluator(IAType reqType, IAType inputType, IScalarEvaluator argEvaluator) {
+ this(reqType, inputType, argEvaluator, null);
+ }
+
+ public CastTypeEvaluator(IAType reqType, IAType inputType, IScalarEvaluator argEvaluator,
+ SourceLocation sourceLoc) {
+ this.sourceLoc = sourceLoc;
resetAndAllocate(reqType, inputType, argEvaluator);
}
@@ -61,7 +75,7 @@
}
protected ACastVisitor createCastVisitor() {
- return new ACastVisitor();
+ return new ACastVisitor(sourceLoc);
}
@Override
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxDescriptor.java
index bdfe7a4..b23037c 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxDescriptor.java
@@ -97,7 +97,8 @@
@Override
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
- return new CastTypeLaxEvaluator(reqType, inputType, recordEvalFactory.createScalarEvaluator(ctx));
+ IScalarEvaluator argEval = recordEvalFactory.createScalarEvaluator(ctx);
+ return new CastTypeLaxEvaluator(reqType, inputType, argEval, sourceLoc);
}
};
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxEvaluator.java
index 35335c6..be74580 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxEvaluator.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeLaxEvaluator.java
@@ -24,6 +24,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -35,13 +36,13 @@
private static final byte[] MISSING_BYTES = new byte[] { ATypeTag.SERIALIZED_MISSING_TYPE_TAG };
- CastTypeLaxEvaluator(IAType reqType, IAType inputType, IScalarEvaluator argEvaluator) {
- super(reqType, inputType, argEvaluator);
+ CastTypeLaxEvaluator(IAType reqType, IAType inputType, IScalarEvaluator argEvaluator, SourceLocation sourceLoc) {
+ super(reqType, inputType, argEvaluator, sourceLoc);
}
@Override
protected ACastVisitor createCastVisitor() {
- return new ACastVisitor(false);
+ return new ACastVisitor(false, sourceLoc);
}
@Override
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index d526d34..0d8557f 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@ -550,6 +550,7 @@
import org.apache.asterix.runtime.runningaggregates.std.RankRunningAggregateDescriptor;
import org.apache.asterix.runtime.runningaggregates.std.RowNumberRunningAggregateDescriptor;
import org.apache.asterix.runtime.runningaggregates.std.TidRunningAggregateDescriptor;
+import org.apache.asterix.runtime.runningaggregates.std.WinMarkFirstMissingRunningAggregateDescriptor;
import org.apache.asterix.runtime.runningaggregates.std.WinPartitionLenRunningAggregateDescriptor;
import org.apache.asterix.runtime.unnestingfunctions.std.RangeDescriptor;
import org.apache.asterix.runtime.unnestingfunctions.std.ScanCollectionDescriptor;
@@ -825,6 +826,7 @@
fc.add(RankRunningAggregateDescriptor.FACTORY);
fc.add(RowNumberRunningAggregateDescriptor.FACTORY);
fc.add(PercentRankRunningAggregateDescriptor.FACTORY);
+ fc.add(WinMarkFirstMissingRunningAggregateDescriptor.FACTORY);
fc.add(WinPartitionLenRunningAggregateDescriptor.FACTORY);
// boolean functions
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/WinMarkFirstMissingRunningAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/WinMarkFirstMissingRunningAggregateDescriptor.java
new file mode 100644
index 0000000..3f5f9f6
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/WinMarkFirstMissingRunningAggregateDescriptor.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.runningaggregates.std;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.runningaggregates.base.AbstractRunningAggregateFunctionDynamicDescriptor;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This internal window function returns {@code TRUE} in the following two cases:
+ * <ol>
+ * <li>the argument is not MISSING</li>
+ * <li>the argument is MISSING and it comes from the first tuple in the current window partition</li>
+ * </ol>
+ * In all other cases the function returns {@code FALSE}.
+ * <p>
+ * The underlying assumption is that tuples in each window partition are sorted on the function's argument in the
+ * descending order.
+ */
+public final class WinMarkFirstMissingRunningAggregateDescriptor
+ extends AbstractRunningAggregateFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY = WinMarkFirstMissingRunningAggregateDescriptor::new;
+
+ @Override
+ public IRunningAggregateEvaluatorFactory createRunningAggregateEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ return new IRunningAggregateEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IRunningAggregateEvaluator createRunningAggregateEvaluator(IEvaluatorContext ctx)
+ throws HyracksDataException {
+ IScalarEvaluator[] evals = new IScalarEvaluator[args.length];
+ for (int i = 0; i < args.length; i++) {
+ evals[i] = args[i].createScalarEvaluator(ctx);
+ }
+ return new WinMarkFirstMissingRunningAggregateEvaluator(evals);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.WIN_MARK_FIRST_MISSING_IMPL;
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/WinMarkFirstMissingRunningAggregateEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/WinMarkFirstMissingRunningAggregateEvaluator.java
new file mode 100644
index 0000000..c872921
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/WinMarkFirstMissingRunningAggregateEvaluator.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.runningaggregates.std;
+
+import java.io.DataOutput;
+
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IWindowAggregateEvaluator;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.TaggedValuePointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public final class WinMarkFirstMissingRunningAggregateEvaluator implements IWindowAggregateEvaluator {
+
+ @SuppressWarnings({ "rawtypes" })
+ private final ISerializerDeserializer boolSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
+
+ private final IScalarEvaluator[] argEvals;
+
+ private final TaggedValuePointable argValue;
+
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+
+ private final DataOutput dataOutput = resultStorage.getDataOutput();
+
+ private boolean first;
+
+ private boolean firstAllMissing;
+
+ WinMarkFirstMissingRunningAggregateEvaluator(IScalarEvaluator[] argEvals) {
+ this.argEvals = argEvals;
+ argValue = TaggedValuePointable.FACTORY.createPointable();
+ }
+
+ @Override
+ public void init() {
+ }
+
+ @Override
+ public void initPartition(long partitionLength) {
+ first = true;
+ }
+
+ @Override
+ public void step(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ boolean value = compute(tuple);
+ resultStorage.reset();
+ boolSerde.serialize(ABoolean.valueOf(value), dataOutput);
+ result.set(resultStorage);
+ }
+
+ private boolean compute(IFrameTupleReference tuple) throws HyracksDataException {
+ if (first) {
+ firstAllMissing = everyArgIsMissing(tuple);
+ first = false;
+ return true;
+ } else {
+ boolean thisAllMissing = firstAllMissing || everyArgIsMissing(tuple);
+ return !thisAllMissing;
+ }
+ }
+
+ private boolean everyArgIsMissing(IFrameTupleReference tuple) throws HyracksDataException {
+ for (IScalarEvaluator argEval : argEvals) {
+ argEval.evaluate(tuple, argValue);
+ if (argValue.getTag() != ATypeTag.SERIALIZED_MISSING_TYPE_TAG) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index e170779..31f5171 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -72,6 +72,7 @@
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.LocalResource;
import org.apache.hyracks.util.ExitUtil;
+import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -195,8 +196,7 @@
try {
createResourceFileMask(resourceFile);
byte[] bytes = OBJECT_MAPPER.writeValueAsBytes(resource.toJson(persistedResourceRegistry));
- final Path path = Paths.get(resourceFile.getAbsolutePath());
- Files.write(path, bytes);
+ FileUtil.writeAndForce(Paths.get(resourceFile.getAbsolutePath()), bytes);
indexCheckpointManagerProvider.get(DatasetResourceReference.of(resource)).init(Long.MIN_VALUE, 0,
LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID.getMaxId());
deleteResourceFileMask(resourceFile);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
index 81002be..a07ca6d 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
@@ -26,7 +26,6 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -42,6 +41,8 @@
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.api.util.IoUtil;
+import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -172,8 +173,11 @@
}
// Write checkpoint file to disk
try {
+ if (path.toFile().exists()) {
+ IoUtil.delete(path);
+ }
byte[] bytes = OBJECT_MAPPER.writeValueAsBytes(checkpoint.toJson(persistedResourceRegistry));
- Files.write(path, bytes, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+ FileUtil.writeAndForce(path, bytes);
readCheckpoint(path);
} catch (IOException e) {
LOGGER.log(Level.ERROR, "Failed to write checkpoint to disk", e);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
index c98489b..3f5012a 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -451,4 +451,14 @@
assignExprList.remove(i);
}
}
+
+ public static int findChild(ILogicalOperator op, LogicalOperatorTag childOpTag) {
+ List<Mutable<ILogicalOperator>> inputs = op.getInputs();
+ for (int i = 0, ln = inputs.size(); i < ln; i++) {
+ if (inputs.get(i).getValue().getOperatorTag() == childOpTag) {
+ return i;
+ }
+ }
+ return -1;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 0f857bd..591cf8c 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -69,7 +69,8 @@
public static final String APPLICATION_JSON = "application/json";
public static final String JSON = "json";
public static final String APPLICATION_X_WWW_FORM_URLENCODED = "application/x-www-form-urlencoded";
- public static final String CSV = "text/csv";
+ public static final String CSV = "csv";
+ public static final String TEXT_CSV = "text/csv";
public static final String IMG_PNG = "image/png";
public static final String TEXT_HTML = "text/html";
public static final String TEXT_PLAIN = "text/plain";
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
index 443640a..fe60653 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
@@ -20,6 +20,8 @@
import java.io.File;
import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.file.Path;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
@@ -86,4 +88,11 @@
}
}
}
+
+ public static void writeAndForce(Path path, byte[] data) throws IOException {
+ try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "rw");) {
+ raf.write(data);
+ raf.getChannel().force(true);
+ }
+ }
}