[ASTERIXDB-2999][RT] Round towards zero when convering floating-point to integer
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Remove use of Math.floor() during floating-point to integer conversion.
The conversion code now rounds both positivie and negative values towards zero
- Update index selection code to account for this change
- Add SqlppNumericIndexRQGTest to randomly test different combinations
of indexes on numeric fields
Change-Id: I1c92d6235e16e6c5d9dadd7a09cd9703781eadb6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14525
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index e532187..3b39d9d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -568,6 +568,25 @@
return primaryKeyVars;
}
+ public static class SearchKeyRoundingFunctionProvider {
+ public TypeCastingMathFunctionType getRoundingFunction(ComparisonKind cKind, Index chosenIndex,
+ IAType indexedFieldType, IAObject constantValue, boolean realTypeConvertedToIntegerType)
+ throws CompilationException {
+ switch (cKind) {
+ case LT:
+ case GE:
+ // round-up
+ return TypeCastingMathFunctionType.CEIL;
+ case LE:
+ case GT:
+ // round-down
+ return TypeCastingMathFunctionType.FLOOR;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, cKind.toString());
+ }
+ }
+ }
+
/**
* Returns the search key expression which feeds a secondary-index search. If we are optimizing a selection query
* then this method returns the a ConstantExpression from the first constant value in the optimizable function
@@ -578,8 +597,8 @@
* @throws AlgebricksException
*/
public static Triple<ILogicalExpression, ILogicalExpression, Boolean> createSearchKeyExpr(Index index,
- IOptimizableFuncExpr optFuncExpr, IAType indexedFieldType, OptimizableOperatorSubTree probeSubTree)
- throws AlgebricksException {
+ IOptimizableFuncExpr optFuncExpr, IAType indexedFieldType, OptimizableOperatorSubTree probeSubTree,
+ SearchKeyRoundingFunctionProvider roundingFunctionProvider) throws AlgebricksException {
SourceLocation sourceLoc = optFuncExpr.getFuncExpr().getSourceLocation();
if (probeSubTree == null) {
// We are optimizing a selection query. Search key is a constant.
@@ -655,18 +674,15 @@
switch (cKind) {
case LT:
- case GE:
- // round-up
- replacedConstantValue = getReplacedConstantValue(constantValue.getObject(),
- constantValueTag, indexedFieldTypeTag, index.isEnforced(),
- TypeCastingMathFunctionType.CEIL, sourceLoc);
- break;
case LE:
case GT:
- // round-down
- replacedConstantValue = getReplacedConstantValue(constantValue.getObject(),
- constantValueTag, indexedFieldTypeTag, index.isEnforced(),
- TypeCastingMathFunctionType.FLOOR, sourceLoc);
+ case GE:
+ TypeCastingMathFunctionType roundingFunction =
+ roundingFunctionProvider.getRoundingFunction(cKind, index, indexedFieldType,
+ constantValue.getObject(), realTypeConvertedToIntegerType);
+ replacedConstantValue =
+ getReplacedConstantValue(constantValue.getObject(), constantValueTag,
+ indexedFieldTypeTag, index.isEnforced(), roundingFunction, sourceLoc);
break;
case EQ:
// equality case - both CEIL and FLOOR need to be applied.
@@ -1162,7 +1178,8 @@
AccessMethodAnalysisContext analysisCtx, OptimizableOperatorSubTree subTree,
LogicalVariable newMissingPlaceHolderForLOJ, IAlgebricksConstantValue leftOuterMissingValue,
List<LogicalVariable> pkVarsFromSIdxUnnestMapOp, List<LogicalVariable> primaryIndexUnnestVars,
- List<Object> primaryIndexOutputTypes) throws AlgebricksException {
+ List<Object> primaryIndexOutputTypes, boolean anyRealTypeConvertedToIntegerType)
+ throws AlgebricksException {
SourceLocation sourceLoc = inputOp.getSourceLocation();
Quadruple<Boolean, Boolean, Boolean, Boolean> indexOnlyPlanInfo = analysisCtx.getIndexOnlyPlanInfo();
// From now on, we deal with the index-only plan.
@@ -1611,7 +1628,8 @@
// other than joining fields, then those conditions need to be applied to filter out
// false positive results in the right path.
// (e.g., where $a.authors /*+ indexnl */ = $b.authors and $a.id = $b.id <- authors:SK, id:PK)
- if ((idxType == IndexType.RTREE || uniqueUsedVarsInTopOp.size() > 1) && requireVerificationAfterSIdxSearch) {
+ if (((idxType == IndexType.RTREE || uniqueUsedVarsInTopOp.size() > 1) && requireVerificationAfterSIdxSearch)
+ || anyRealTypeConvertedToIntegerType) {
// Creates a new SELECT operator by deep-copying the SELECT operator in the left path
// since we need to change the variable reference in the SELECT operator.
// For the index-nested-loop join case, we copy the condition of the join operator.
@@ -1734,8 +1752,8 @@
IOptimizationContext context, boolean sortPrimaryKeys, boolean retainInput, boolean retainMissing,
boolean requiresBroadcast, Index secondaryIndex, AccessMethodAnalysisContext analysisCtx,
OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree,
- LogicalVariable newMissingPlaceHolderForLOJ, IAlgebricksConstantValue leftOuterMissingValue)
- throws AlgebricksException {
+ LogicalVariable newMissingPlaceHolderForLOJ, IAlgebricksConstantValue leftOuterMissingValue,
+ boolean anyRealTypeConvertedToIntegerType) throws AlgebricksException {
// Common part for the non-index-only plan and index-only plan
// Variables and types for the primary-index search.
List<LogicalVariable> primaryIndexUnnestVars = new ArrayList<>();
@@ -1788,7 +1806,8 @@
return createFinalIndexOnlySearchPlan(afterTopOpRefs, topOpRef, conditionRef, assignsBeforeTopOpRef,
dataset, recordType, metaRecordType, inputOp, context, retainInput, retainMissing,
requiresBroadcast, secondaryIndex, analysisCtx, indexSubTree, newMissingPlaceHolderForLOJ,
- leftOuterMissingValue, pkVarsFromSIdxUnnestMapOp, primaryIndexUnnestVars, primaryIndexOutputTypes);
+ leftOuterMissingValue, pkVarsFromSIdxUnnestMapOp, primaryIndexUnnestVars, primaryIndexOutputTypes,
+ anyRealTypeConvertedToIntegerType);
} else {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, inputOp.getSourceLocation(),
"Cannot use index-only plan with array indexes.");
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
index ce25d4d..d01a2cb 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
@@ -227,23 +227,6 @@
}
@Override
- protected IAType getIndexedKeyType(Index.IIndexDetails chosenIndexDetails, int keyPos) throws CompilationException {
- Index.ArrayIndexDetails arrayIndexDetails = (Index.ArrayIndexDetails) chosenIndexDetails;
- int elementPos = 0;
- for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
- for (int i = 0; i < e.getProjectList().size(); i++) {
- if (elementPos == keyPos) {
- return e.getTypeList().get(i);
- }
- elementPos++;
- }
- }
-
- throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE,
- "No array index element found, but using " + "an array access method.");
- }
-
- @Override
public boolean matchIndexType(IndexType indexType) {
return indexType == IndexType.ARRAY;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 60dd53b..5dd0135 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -39,17 +39,21 @@
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.dataflow.data.common.ILogicalBinaryComparator;
+import org.apache.asterix.dataflow.data.nontagged.comparators.ComparatorUtil;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.metadata.utils.TypeUtil;
+import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
@@ -379,6 +383,7 @@
boolean couldntFigureOut = false;
boolean doneWithExprs = false;
boolean isEqCondition = false;
+ boolean anyRealTypeConvertedToIntegerType = false;
BitSet setLowKeys = new BitSet(numSecondaryKeys);
BitSet setHighKeys = new BitSet(numSecondaryKeys);
// Go through the func exprs listed as optimizable by the chosen index,
@@ -406,10 +411,12 @@
// This is required because of type-casting. Refer to AccessMethodUtils.createSearchKeyExpr for details.
IAType indexedFieldType = chosenIndexKeyFieldTypes.get(keyPos);
Triple<ILogicalExpression, ILogicalExpression, Boolean> returnedSearchKeyExpr =
- AccessMethodUtils.createSearchKeyExpr(chosenIndex, optFuncExpr, indexedFieldType, probeSubTree);
+ AccessMethodUtils.createSearchKeyExpr(chosenIndex, optFuncExpr, indexedFieldType, probeSubTree,
+ SEARCH_KEY_ROUNDING_FUNCTION_COMPUTER);
ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
ILogicalExpression searchKeyEQExpr = null;
boolean realTypeConvertedToIntegerType = returnedSearchKeyExpr.third;
+ anyRealTypeConvertedToIntegerType |= realTypeConvertedToIntegerType;
LimitType limit = getLimitType(optFuncExpr, probeSubTree);
if (limit == null) {
@@ -423,7 +430,7 @@
}
// Deals with the non-enforced index case here.
- if (relaxLimitTypeToInclusive(chosenIndex, keyPos, realTypeConvertedToIntegerType)) {
+ if (relaxLimitTypeToInclusive(chosenIndex, indexedFieldType, realTypeConvertedToIntegerType)) {
if (limit == LimitType.HIGH_EXCLUSIVE) {
limit = LimitType.HIGH_INCLUSIVE;
} else if (limit == LimitType.LOW_EXCLUSIVE) {
@@ -688,7 +695,8 @@
indexSearchOp = AccessMethodUtils.createRestOfIndexSearchPlan(afterTopOpRefs, topOpRef, conditionRef,
assignBeforeTheOpRefs, dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp,
context, true, retainInput, retainMissing, false, chosenIndex, analysisCtx, indexSubTree,
- probeSubTree, newMissingNullPlaceHolderForLOJ, leftOuterMissingValue);
+ probeSubTree, newMissingNullPlaceHolderForLOJ, leftOuterMissingValue,
+ anyRealTypeConvertedToIntegerType);
// Replaces the datasource scan with the new plan rooted at
// Get dataSourceRef operator -
@@ -914,8 +922,41 @@
return limit;
}
- private boolean relaxLimitTypeToInclusive(Index chosenIndex, int keyPos, boolean realTypeConvertedToIntegerType)
- throws CompilationException {
+ private static final BTreeSearchKeyRoundingFunctionProvider SEARCH_KEY_ROUNDING_FUNCTION_COMPUTER =
+ new BTreeSearchKeyRoundingFunctionProvider();
+
+ private static class BTreeSearchKeyRoundingFunctionProvider
+ extends AccessMethodUtils.SearchKeyRoundingFunctionProvider {
+
+ private final ILogicalBinaryComparator DOUBLE_CMPR =
+ ComparatorUtil.createLogicalComparator(BuiltinType.ADOUBLE, BuiltinType.ADOUBLE, false);
+
+ private static final ADouble ZERO = new ADouble(0);
+
+ @Override
+ public ATypeHierarchy.TypeCastingMathFunctionType getRoundingFunction(ComparisonKind cKind, Index chosenIndex,
+ IAType indexedFieldType, IAObject constantValue, boolean realTypeConvertedToIntegerType)
+ throws CompilationException {
+ switch (cKind) {
+ case GE:
+ return relaxLimitTypeToInclusive(chosenIndex, indexedFieldType, realTypeConvertedToIntegerType)
+ && DOUBLE_CMPR.compare(ZERO, constantValue) == ILogicalBinaryComparator.Result.LT
+ ? ATypeHierarchy.TypeCastingMathFunctionType.FLOOR
+ : ATypeHierarchy.TypeCastingMathFunctionType.CEIL;
+ case LE:
+ return relaxLimitTypeToInclusive(chosenIndex, indexedFieldType, realTypeConvertedToIntegerType)
+ && DOUBLE_CMPR.compare(ZERO, constantValue) == ILogicalBinaryComparator.Result.GT
+ ? ATypeHierarchy.TypeCastingMathFunctionType.CEIL
+ : ATypeHierarchy.TypeCastingMathFunctionType.FLOOR;
+ default:
+ return super.getRoundingFunction(cKind, chosenIndex, indexedFieldType, constantValue,
+ realTypeConvertedToIntegerType);
+ }
+ }
+ }
+
+ private static boolean relaxLimitTypeToInclusive(Index chosenIndex, IAType indexedFieldType,
+ boolean realTypeConvertedToIntegerType) {
// For a non-enforced index or an enforced index that stores a casted value on the given index,
// we need to apply the following transformation.
// For an index on a closed field, this transformation is not necessary since the value between
@@ -943,11 +984,7 @@
}
if (chosenIndex.getIndexDetails().isOverridingKeyFieldTypes() && !chosenIndex.isEnforced()) {
- IAType indexedKeyType = getIndexedKeyType(chosenIndex.getIndexDetails(), keyPos);
- if (NonTaggedFormatUtil.isOptional(indexedKeyType)) {
- indexedKeyType = ((AUnionType) indexedKeyType).getActualType();
- }
- switch (indexedKeyType.getTypeTag()) {
+ switch (TypeComputeUtils.getActualType(indexedFieldType).getTypeTag()) {
case TINYINT:
case SMALLINT:
case INTEGER:
@@ -963,10 +1000,6 @@
return false;
}
- protected IAType getIndexedKeyType(Index.IIndexDetails chosenIndexDetails, int keyPos) throws CompilationException {
- return ((Index.ValueIndexDetails) chosenIndexDetails).getKeyFieldTypes().get(keyPos);
- }
-
private boolean probeIsOnLhs(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree probeSubTree) {
if (probeSubTree == null) {
if (optFuncExpr.getConstantExpressions().length == 0) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index ff3d887..62812aa 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -461,7 +461,7 @@
ILogicalOperator primaryIndexUnnestOp = AccessMethodUtils.createRestOfIndexSearchPlan(afterTopOpRefs, topOpRef,
conditionRef, assignBeforeTopOpRefs, dataSourceScan, dataset, recordType, metaRecordType,
secondaryIndexUnnestOp, context, true, retainInput, retainNull, false, chosenIndex, analysisCtx,
- indexSubTree, null, newMissingNullPlaceHolderForLOJ, leftOuterMissingValue);
+ indexSubTree, null, newMissingNullPlaceHolderForLOJ, leftOuterMissingValue, false);
return primaryIndexUnnestOp;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 971760a..6959543 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -44,6 +44,7 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Quadruple;
+import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -216,6 +217,9 @@
return true;
}
+ private static final AccessMethodUtils.SearchKeyRoundingFunctionProvider SEARCH_KEY_ROUNDING_FUNCTION_PROVIDER =
+ new AccessMethodUtils.SearchKeyRoundingFunctionProvider();
+
@Override
public ILogicalOperator createIndexSearchPlan(List<Mutable<ILogicalOperator>> afterTopRefs,
Mutable<ILogicalOperator> topRef, Mutable<ILogicalExpression> conditionRef,
@@ -263,8 +267,9 @@
ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
// List of expressions for the assign.
ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
- ILogicalExpression returnedSearchKeyExpr =
- AccessMethodUtils.createSearchKeyExpr(chosenIndex, optFuncExpr, optFieldType, probeSubTree).first;
+ Triple<ILogicalExpression, ILogicalExpression, Boolean> returnedSearchKeyExpr =
+ AccessMethodUtils.createSearchKeyExpr(chosenIndex, optFuncExpr, optFieldType, probeSubTree,
+ SEARCH_KEY_ROUNDING_FUNCTION_PROVIDER);
for (int i = 0; i < numSecondaryKeys; i++) {
// The create MBR function "extracts" one field of an MBR around the given spatial object.
@@ -272,7 +277,7 @@
new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR));
createMBR.setSourceLocation(optFuncExpr.getFuncExpr().getSourceLocation());
// Spatial object is the constant from the func expr we are optimizing.
- createMBR.getArguments().add(new MutableObject<>(returnedSearchKeyExpr.cloneExpression()));
+ createMBR.getArguments().add(new MutableObject<>(returnedSearchKeyExpr.first.cloneExpression()));
// The number of dimensions
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
new ConstantExpression(new AsterixConstantValue(new AInt32(numDimensions)))));
@@ -315,7 +320,7 @@
: AccessMethodUtils.createRestOfIndexSearchPlan(afterTopRefs, topRef, conditionRef, assignBeforeTopRefs,
dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true,
retainInput, retainNull, false, chosenIndex, analysisCtx, indexSubTree, null,
- newMissingNullPlaceHolderForLOJ, leftOuterMissingValue);
+ newMissingNullPlaceHolderForLOJ, leftOuterMissingValue, returnedSearchKeyExpr.third);
}
@Override
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppNumericIndexRQGTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppNumericIndexRQGTest.java
new file mode 100644
index 0000000..5201681
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppNumericIndexRQGTest.java
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.runtime;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.asterix.api.http.server.QueryServiceRequestParameters;
+import org.apache.asterix.common.config.CompilerProperties;
+import org.apache.asterix.common.utils.Servlets;
+import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.lang.sqlpp.parser.SqlppHint;
+import org.apache.asterix.metadata.utils.TypeUtil;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.asterix.testframework.xml.ParameterTypeEnum;
+import org.apache.asterix.testframework.xml.TestCase;
+import org.apache.commons.math3.random.MersenneTwister;
+import org.apache.commons.math3.random.RandomGenerator;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * RQG testsuite for indexes on numeric fields
+ */
+@RunWith(Parameterized.class)
+public class SqlppNumericIndexRQGTest {
+
+ static final Logger LOGGER = LogManager.getLogger(SqlppNumericIndexRQGTest.class);
+
+ static final String CONF_PROPERTY_SEED =
+ SqlppRQGTestBase.getConfigurationPropertyName(SqlppNumericIndexRQGTest.class, "seed");
+ static final long CONF_PROPERTY_SEED_DEFAULT = System.currentTimeMillis();
+
+ static final String CONF_PROPERTY_LIMIT =
+ SqlppRQGTestBase.getConfigurationPropertyName(SqlppNumericIndexRQGTest.class, "limit");
+ static final int CONF_PROPERTY_LIMIT_DEFAULT = 50;
+
+ static final String CONF_PROPERTY_OFFSET =
+ SqlppRQGTestBase.getConfigurationPropertyName(SqlppNumericIndexRQGTest.class, "offset");
+ static final int CONF_PROPERTY_OFFSET_DEFAULT = 0;
+
+ static final String CONF_PROPERTY_OFFSET_QUERY =
+ SqlppRQGTestBase.getConfigurationPropertyName(SqlppNumericIndexRQGTest.class, "offset.query");
+ static final int CONF_PROPERTY_OFFSET_QUERY_DEFAULT = 0;
+
+ static final Path OUTPUT_DIR = Paths.get("target", SqlppNumericIndexRQGTest.class.getSimpleName());
+
+ static final String DATAVERSE_NAME = "dvTest";
+ static final String DATASET_NAME_TYPED = "dsTyped";
+ static final String DATASET_NAME_UNTYPED = "dsUntyped";
+ static final String COMPILER_OPTION_FORMAT = "set `%s` '%s';";
+ static final String ID_COLUMN_NAME = "id";
+
+ static final List<AlgebricksBuiltinFunctions.ComparisonKind> CMP_KINDS =
+ Arrays.asList(AlgebricksBuiltinFunctions.ComparisonKind.GT, AlgebricksBuiltinFunctions.ComparisonKind.GE,
+ AlgebricksBuiltinFunctions.ComparisonKind.LT, AlgebricksBuiltinFunctions.ComparisonKind.LE,
+ AlgebricksBuiltinFunctions.ComparisonKind.EQ);
+
+ static final List<BuiltinType> FIELD_TYPES = Arrays.asList(BuiltinType.ANY, BuiltinType.AINT8, BuiltinType.AINT16,
+ BuiltinType.AINT32, BuiltinType.AINT64, BuiltinType.AFLOAT, BuiltinType.ADOUBLE);
+
+ static final double MIN_VALUE = -3.5;
+ static final double MAX_VALUE = 3.5;
+ static final double PROBE_STEP = 1.0 / 4;
+ static final double DATA_STEP = PROBE_STEP / 2;
+ static final int NON_NUMERIC_DATA_RATIO = 8;
+ static final int INDEX_ONLY_ENABLED_RATIO = 3;
+ static final int INDEXED_AS_IS_RATIO = 4;
+ static final int JOIN_RATIO = 8;
+
+ static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ static final ObjectReader OBJECT_READER = OBJECT_MAPPER.readerFor(ObjectNode.class);
+
+ static TestExecutor testExecutor;
+
+ private final TestInstance testInstance;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ testExecutor = new TestExecutor();
+ LangExecutionUtil.setUp(SqlppRQGTestBase.TEST_CONFIG_FILE_NAME, testExecutor);
+
+ StringBuilder sb = new StringBuilder(2048);
+ addDropDataverse(sb, DATAVERSE_NAME);
+ addCreateDataverse(sb, DATAVERSE_NAME);
+ addCreateDataset(sb, DATAVERSE_NAME, DATASET_NAME_TYPED, false);
+ addLoadDataset(sb, DATAVERSE_NAME, DATASET_NAME_TYPED);
+ addCreateDataset(sb, DATAVERSE_NAME, DATASET_NAME_UNTYPED, true);
+ addLoadDataset(sb, DATAVERSE_NAME, DATASET_NAME_UNTYPED);
+ executeUpdateOrDdl(sb.toString());
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameterized.Parameters(name = "SqlppNumericIndexRQGTest {index}: {0}")
+ public static Collection<TestInstance> tests() {
+
+ long seed = SqlppRQGTestBase.getLongConfigurationProperty(CONF_PROPERTY_SEED, CONF_PROPERTY_SEED_DEFAULT);
+ int limit =
+ (int) SqlppRQGTestBase.getLongConfigurationProperty(CONF_PROPERTY_LIMIT, CONF_PROPERTY_LIMIT_DEFAULT);
+ int testOffset =
+ (int) SqlppRQGTestBase.getLongConfigurationProperty(CONF_PROPERTY_OFFSET, CONF_PROPERTY_OFFSET_DEFAULT);
+ int queryOffset = (int) SqlppRQGTestBase.getLongConfigurationProperty(CONF_PROPERTY_OFFSET_QUERY,
+ CONF_PROPERTY_OFFSET_QUERY_DEFAULT);
+
+ LOGGER.info(String.format("Testsuite configuration: -D%s=%d -D%s=%d -D%s=%d -D%s=%d", CONF_PROPERTY_SEED, seed,
+ CONF_PROPERTY_LIMIT, limit, CONF_PROPERTY_OFFSET, testOffset, CONF_PROPERTY_OFFSET_QUERY, queryOffset));
+
+ String indexName = "idx_" + seed;
+
+ List<TestInstance> testCases = new ArrayList<>(limit);
+ RandomGenerator random = new MersenneTwister(seed);
+
+ int i = 0;
+ while (i < limit) {
+ BuiltinType fieldType = randomElement(FIELD_TYPES, random);
+
+ BuiltinType indexedType;
+ if (random.nextInt(INDEXED_AS_IS_RATIO) == 0) {
+ if (fieldType.getTypeTag() == ATypeTag.ANY) {
+ continue;
+ }
+ indexedType = null;
+ } else {
+ indexedType = randomElement(FIELD_TYPES, random);
+ if (indexedType.getTypeTag() == ATypeTag.ANY) {
+ continue;
+ }
+ }
+
+ boolean join = random.nextInt(JOIN_RATIO) == 0;
+ BuiltinType probeType;
+ if (join) {
+ probeType = indexedType != null ? indexedType : fieldType;
+ } else {
+ probeType = randomElement(FIELD_TYPES, random);
+ }
+ if (probeType.getTypeTag() == ATypeTag.ANY) {
+ continue;
+ }
+
+ Set<TestOption> options = EnumSet.noneOf(TestOption.class);
+ if (random.nextBoolean()) {
+ options.add(TestOption.EXCLUDE_UNKNOWN_KEY);
+ }
+ if (indexedType != null && random.nextBoolean()) {
+ options.add(TestOption.CAST_DEFAULT_NULL);
+ }
+ if (random.nextInt(INDEX_ONLY_ENABLED_RATIO) == 0) {
+ options.add(TestOption.INDEX_ONLY_ENABLED);
+ }
+ if (join) {
+ options.add(random.nextBoolean() ? TestOption.INNER_JOIN_QUERY : TestOption.LEFT_OUTER_JOIN_QUERY);
+ }
+ String searchedDatasetName = indexedType == null ? DATASET_NAME_TYPED : DATASET_NAME_UNTYPED;
+ String probeSourceDatasetName = indexedType == null ? DATASET_NAME_UNTYPED : DATASET_NAME_TYPED;
+ if (i >= testOffset) {
+ TestInstance testCase = new TestInstance(i, DATAVERSE_NAME, searchedDatasetName, indexName, fieldType,
+ indexedType, probeType, probeSourceDatasetName, options, queryOffset);
+ testCases.add(testCase);
+ }
+ i++;
+ }
+
+ return testCases;
+ }
+
+ public SqlppNumericIndexRQGTest(TestInstance testInstance) {
+ this.testInstance = testInstance;
+ }
+
+ @Test
+ public void test() throws Exception {
+ LOGGER.info(testInstance);
+ testInstance.execute();
+ }
+
+ private static boolean hasIndexSearch(ArrayNode planLines, String indexName) {
+ for (int i = 0, n = planLines.size(); i < n; i++) {
+ String line = planLines.get(i).textValue();
+ if (line.contains(BuiltinFunctions.INDEX_SEARCH.getName()) && line.contains('"' + indexName + '"')) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static void executeUpdateOrDdl(String statement) throws Exception {
+ LOGGER.debug("Executing: " + statement);
+ testExecutor.executeSqlppUpdateOrDdl(statement, TestCaseContext.OutputFormat.CLEAN_JSON);
+ }
+
+ private static Pair<ArrayNode, String> executeQuery(String query, boolean fetchPlan) throws Exception {
+ LOGGER.debug("Executing: " + query);
+
+ List<TestCase.CompilationUnit.Parameter> params;
+ if (fetchPlan) {
+ TestCase.CompilationUnit.Parameter planParameter = new TestCase.CompilationUnit.Parameter();
+ planParameter.setName(QueryServiceRequestParameters.Parameter.OPTIMIZED_LOGICAL_PLAN.str());
+ planParameter.setValue(Boolean.TRUE.toString());
+ planParameter.setType(ParameterTypeEnum.STRING);
+ params = Collections.singletonList(planParameter);
+ } else {
+ params = Collections.emptyList();
+ }
+
+ try (InputStream resultStream = testExecutor.executeQueryService(query, TestCaseContext.OutputFormat.CLEAN_JSON,
+ testExecutor.getEndpoint(Servlets.QUERY_SERVICE), params, true, StandardCharsets.UTF_8)) {
+ JsonNode r = OBJECT_READER.readTree(resultStream);
+ JsonNode errors = r.get("errors");
+ if (errors != null) {
+ Assert.fail("Query failed: " + errors);
+ }
+ JsonNode results = r.get("results");
+ if (!results.isArray()) {
+ Assert.fail("Expected array result, got: " + results);
+ }
+ ArrayNode resultsArray = (ArrayNode) results;
+ String plan = fetchPlan ? r.get("plans").get("optimizedLogicalPlan").asText() : null;
+ return new Pair<>(resultsArray, plan);
+ }
+ }
+
+ private static void addDropDataverse(StringBuilder sb, String dataverseName) {
+ sb.append(String.format("DROP DATAVERSE %s IF EXISTS;\n", dataverseName));
+ }
+
+ private static void addCreateDataverse(StringBuilder sb, String dataverseName) {
+ sb.append(String.format("CREATE DATAVERSE %s;\n", dataverseName));
+ }
+
+ private static void addCreateDataset(StringBuilder sb, String dataverseName, String datasetName, boolean untyped) {
+ sb.append("CREATE DATASET ").append(dataverseName).append('.').append(datasetName);
+ sb.append(" (").append(ID_COLUMN_NAME).append(" string ");
+ if (!untyped) {
+ for (BuiltinType t : FIELD_TYPES) {
+ if (t.getTypeTag() == ATypeTag.ANY) {
+ continue;
+ }
+ sb.append(", ").append(getColumnName(t)).append(' ').append(t.getTypeName());
+ }
+ }
+ sb.append(") ");
+ sb.append("OPEN TYPE PRIMARY KEY id;\n");
+ }
+
+ private static void addLoadDataset(StringBuilder sb, String dataverseName, String datasetName) {
+ int id = 0, nonNumeric = 0;
+ for (double v = MIN_VALUE; v <= MAX_VALUE; v += DATA_STEP) {
+ addInsert(sb, dataverseName, datasetName, id, v);
+ id++;
+
+ // Insert non-numeric values once in a while
+ if (NON_NUMERIC_DATA_RATIO > 0 && id % NON_NUMERIC_DATA_RATIO == 0) {
+ Double nnv;
+ switch (nonNumeric % 3) {
+ case 0:
+ nnv = null;
+ break;
+ case 1:
+ nnv = Double.POSITIVE_INFINITY;
+ break;
+ case 2:
+ nnv = Double.NEGATIVE_INFINITY;
+ break;
+ default:
+ throw new IllegalStateException(String.valueOf(nonNumeric));
+ }
+ addInsert(sb, dataverseName, datasetName, id, nnv);
+ nonNumeric++;
+ id++;
+ }
+ }
+ }
+
+ private static void addInsert(StringBuilder sb, String dataverseName, String datasetName, int id, Double v) {
+ sb.append("INSERT INTO ").append(dataverseName).append('.').append(datasetName).append(" ( { ");
+ sb.append("'").append(ID_COLUMN_NAME).append("': \"").append(datasetName).append(':').append(id).append('"');
+ for (BuiltinType fieldType : FIELD_TYPES) {
+ String columnName = getColumnName(fieldType);
+ sb.append(", '").append(columnName).append("':");
+ if (v == null) {
+ sb.append("null");
+ } else if (Double.isInfinite(v)) {
+ sb.append(fieldType.getTypeTag() == ATypeTag.DOUBLE || fieldType.getTypeTag() == ATypeTag.FLOAT
+ ? String.format("%s('%sINF')", fieldType.getTypeName(), v < 0 ? "-" : "") : "null");
+ } else {
+ BuiltinType valueType;
+ if (fieldType.getTypeTag() == ATypeTag.ANY) {
+ valueType = FIELD_TYPES.get(id % FIELD_TYPES.size());
+ if (valueType.getTypeTag() == ATypeTag.ANY) {
+ valueType = BuiltinType.ASTRING;
+ }
+ } else {
+ valueType = fieldType;
+ }
+ String castFn = valueType.getTypeName();
+ sb.append(castFn).append('(').append(v).append(')');
+ }
+ }
+ sb.append("} );\n");
+ }
+
+ private static String getColumnName(BuiltinType t) {
+ return "c_" + t.getTypeName();
+ }
+
+ private static <T> T randomElement(List<T> list, RandomGenerator randomGenerator) {
+ return list.get(randomGenerator.nextInt(list.size()));
+ }
+
+ private static ArrayNode readLinesIntoArrayNode(String str) {
+ try {
+ ArrayNode arrayNode = OBJECT_MAPPER.createArrayNode();
+ BufferedReader br = new BufferedReader(new StringReader(str));
+ String line;
+ while ((line = br.readLine()) != null) {
+ arrayNode.add(line);
+ }
+ return arrayNode;
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private enum TestOption {
+ EXCLUDE_UNKNOWN_KEY,
+ CAST_DEFAULT_NULL,
+ INDEX_ONLY_ENABLED,
+ JOIN_QUERY,
+ INNER_JOIN_QUERY,
+ LEFT_OUTER_JOIN_QUERY
+
+ }
+
+ private static class TestInstance {
+
+ private final int id;
+
+ private final String dataverseName;
+
+ private final String datasetName;
+
+ private final String indexName;
+
+ private final BuiltinType fieldType;
+
+ private final BuiltinType indexedType;
+
+ private final BuiltinType probeType;
+
+ private final String probeSourceDatasetName; // for JOIN query
+
+ private final Set<TestOption> options;
+
+ private final int queryOffset;
+
+ public TestInstance(int id, String dataverseName, String datasetName, String indexName, BuiltinType fieldType,
+ BuiltinType indexedType, BuiltinType probeType, String probeSourceDatasetName,
+ Set<TestOption> testOptions, int queryOffset) {
+ this.id = id;
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ this.indexName = indexName;
+ this.fieldType = fieldType;
+ this.indexedType = indexedType;
+ this.probeType = probeType;
+ this.probeSourceDatasetName = probeSourceDatasetName;
+ this.options = testOptions == null ? Collections.emptySet() : testOptions;
+ this.queryOffset = queryOffset;
+ }
+
+ void execute() throws Exception {
+ List<String> queries;
+ if (options.contains(TestOption.INNER_JOIN_QUERY)) {
+ queries = Collections.singletonList(generateJoinQuery(false));
+ } else if (options.contains(TestOption.LEFT_OUTER_JOIN_QUERY)) {
+ queries = Collections.singletonList(generateJoinQuery(true));
+ } else {
+ queries = new ArrayList<>((int) ((MAX_VALUE - MIN_VALUE) / PROBE_STEP) + 1);
+ for (double v = MIN_VALUE; v <= MAX_VALUE; v += PROBE_STEP) {
+ for (AlgebricksBuiltinFunctions.ComparisonKind cmpKind : CMP_KINDS) {
+ queries.add(generateSearchQuery(cmpKind, v));
+ }
+ }
+ }
+
+ int queryCount = queries.size();
+
+ executeUpdateOrDdl(generateDropIndex());
+
+ LOGGER.info(String.format("Running queries [%d to %d] without index ...", queryOffset, queryCount - 1));
+ ArrayNode[] resNoIndex = new ArrayNode[queryCount];
+ for (int i = queryOffset; i < queryCount; i++) {
+ String query = queries.get(i);
+ ArrayNode result = executeQuery(query, false).first;
+ resNoIndex[i] = result;
+ }
+
+ LOGGER.info("Creating index");
+ executeUpdateOrDdl(generateCreateIndex());
+
+ LOGGER.info(String.format("Running queries [%d to %d] with index ...", queryOffset, queryCount - 1));
+ for (int i = queryOffset; i < queryCount; i++) {
+ String query = queries.get(i);
+ ArrayNode rNoIndex = resNoIndex[i];
+ Pair<ArrayNode, String> pWithIndex = executeQuery(query, true);
+ ArrayNode planWithIndex = readLinesIntoArrayNode(pWithIndex.second);
+ String comment = String.format("%s;%s", this, query);
+ if (!hasIndexSearch(planWithIndex, indexName)) {
+ File fPlan = SqlppRQGTestBase.writeResult(OUTPUT_DIR, planWithIndex, id, "plan", comment,
+ (out, node) -> out.println(node.asText()));
+ Assert.fail(
+ String.format("Index was not used. Expected to find search of index [%s] in query plan: %s",
+ indexName, fPlan.getAbsolutePath()));
+ }
+ ArrayNode rWithIndex = pWithIndex.first;
+ if (!rNoIndex.equals(rWithIndex)) {
+ File fNoIndex = SqlppRQGTestBase.writeResult(OUTPUT_DIR, rNoIndex, id, "no_index", comment);
+ File fWithIndex = SqlppRQGTestBase.writeResult(OUTPUT_DIR, rWithIndex, id, "with_index", comment);
+ Assert.fail(
+ String.format("Different results for query #%d [%s].%nWithout index: %s%nWith index: %s", i,
+ query, fNoIndex.getAbsolutePath(), fWithIndex.getAbsolutePath()));
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(64);
+ sb.append('#').append(id).append(' ');
+ sb.append(generateCreateIndex(true));
+ sb.append(String.format(" PROBE(%s)", probeType.getTypeName()));
+ if (options.contains(TestOption.INDEX_ONLY_ENABLED)) {
+ sb.append(" INDEX_ONLY");
+ }
+ if (options.contains(TestOption.INNER_JOIN_QUERY)) {
+ sb.append(" JOIN");
+ } else if (options.contains(TestOption.LEFT_OUTER_JOIN_QUERY)) {
+ sb.append(" LEFT JOIN");
+ }
+ return sb.toString();
+ }
+
+ private String generateDropIndex() {
+ return String.format("DROP INDEX %s.%s.%s IF EXISTS", dataverseName, datasetName, indexName);
+ }
+
+ private String generateCreateIndex() {
+ return generateCreateIndex(false);
+ }
+
+ private String generateCreateIndex(boolean displayOnly) {
+ StringBuilder sb = new StringBuilder(64);
+ String columnName = getColumnName(fieldType);
+ if (!displayOnly) {
+ sb.append("CREATE INDEX ").append(indexName).append(" ON ").append(dataverseName).append('.');
+ }
+ sb.append(datasetName).append("(").append(columnName);
+ if (indexedType != null) {
+ sb.append(':').append(indexedType.getTypeName());
+ }
+ sb.append(")");
+ sb.append(" ");
+ if (options.contains(TestOption.EXCLUDE_UNKNOWN_KEY)) {
+ sb.append(displayOnly ? "-UNKN_KEY" : "EXCLUDE UNKNOWN KEY");
+ } else {
+ sb.append(displayOnly ? "+UNKN_KEY" : "INCLUDE UNKNOWN KEY");
+ }
+ if (options.contains(TestOption.CAST_DEFAULT_NULL)) {
+ sb.append(" CAST(DEFAULT NULL)");
+ }
+ return sb.toString();
+ }
+
+ private String generateSearchQuery(AlgebricksBuiltinFunctions.ComparisonKind cmpKind, double p) {
+ String columnName = getColumnName(fieldType);
+ boolean castDefaultNull = options.contains(TestOption.CAST_DEFAULT_NULL);
+
+ StringBuilder sb = new StringBuilder(128);
+ addQueryProlog(sb);
+
+ String indexedFieldCastFn = castDefaultNull ? getTypeConstructorFunction(indexedType) : "";
+ String probeValueCastFn = probeType.getTypeName();
+
+ sb.append(String.format("SELECT VALUE %s ", columnName));
+ sb.append(String.format("FROM %s.%s ", dataverseName, datasetName));
+ sb.append(String.format("WHERE %s(%s) %s %s(%s) ", indexedFieldCastFn, columnName, getCmpOp(cmpKind),
+ probeValueCastFn, p));
+ sb.append(String.format("ORDER BY %s", columnName));
+ return sb.toString();
+ }
+
+ private String generateJoinQuery(boolean leftOuterJoin) {
+ if (!canJoin(fieldType, indexedType, probeType)) {
+ throw new IllegalStateException();
+ }
+
+ String columnName = getColumnName(fieldType);
+ String cmpOp = getCmpOp(AlgebricksBuiltinFunctions.ComparisonKind.EQ);
+ boolean castDefaultNull = options.contains(TestOption.CAST_DEFAULT_NULL);
+
+ StringBuilder sb = new StringBuilder(128);
+ addQueryProlog(sb);
+
+ String indexedFieldCastFn = castDefaultNull ? getTypeConstructorFunction(indexedType) : "";
+ String probeValueCastFn = probeType.getTypeName();
+
+ sb.append(String.format("SELECT t1.%s AS t1_%s, t2.%s AS t2_%s ", ID_COLUMN_NAME, ID_COLUMN_NAME,
+ ID_COLUMN_NAME, ID_COLUMN_NAME));
+ sb.append(String.format("FROM %s.%s t1 ", dataverseName, probeSourceDatasetName));
+ sb.append(String.format("%s JOIN %s.%s t2 ", leftOuterJoin ? "LEFT" : "", dataverseName, datasetName));
+ sb.append(String.format("ON %s(t1.%s) /* +%s */ %s %s(t2.%s) ", probeValueCastFn, columnName,
+ SqlppHint.INDEXED_NESTED_LOOP_JOIN_HINT.getIdentifier(), cmpOp, indexedFieldCastFn, columnName));
+ sb.append(String.format("ORDER BY t1_%s, t2_%s", ID_COLUMN_NAME, ID_COLUMN_NAME));
+ return sb.toString();
+ }
+
+ private void addQueryProlog(StringBuilder sb) {
+ sb.append(String.format(COMPILER_OPTION_FORMAT, CompilerProperties.COMPILER_SORT_PARALLEL_KEY, false));
+ sb.append(String.format(COMPILER_OPTION_FORMAT, CompilerProperties.COMPILER_INDEXONLY_KEY,
+ options.contains(TestOption.INDEX_ONLY_ENABLED)));
+ if (options.contains(TestOption.CAST_DEFAULT_NULL)) {
+ sb.append(String.format(COMPILER_OPTION_FORMAT, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, true));
+ }
+ }
+
+ private String getTypeConstructorFunction(BuiltinType probeType) {
+ return options.contains(TestOption.CAST_DEFAULT_NULL)
+ ? Objects.requireNonNull(TypeUtil.getTypeConstructorDefaultNull(probeType, false)).getName()
+ .replace('-', '_')
+ : probeType.getTypeName();
+ }
+
+ private String getCmpOp(AlgebricksBuiltinFunctions.ComparisonKind cmpKind) {
+ switch (cmpKind) {
+ case GT:
+ return ">";
+ case GE:
+ return ">=";
+ case LE:
+ return "<";
+ case LT:
+ return "<=";
+ case EQ:
+ return "=";
+ case NEQ:
+ default:
+ throw new IllegalStateException(String.valueOf(cmpKind));
+ }
+ }
+
+ private static boolean canJoin(BuiltinType fieldType, BuiltinType indexedType, BuiltinType probeType) {
+ return probeType.getTypeTag() == (indexedType != null ? indexedType : fieldType).getTypeTag();
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppRQGTestBase.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppRQGTestBase.java
index aa8783a..ec4b55f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppRQGTestBase.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppRQGTestBase.java
@@ -70,9 +70,6 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
-// Prerequisite:
-// setenv TESTCONTAINERS_RYUK_DISABLED true
-
public abstract class SqlppRQGTestBase {
private static final Logger LOGGER = LogManager.getLogger(SqlppRQGTestBase.class);
@@ -167,8 +164,8 @@
boolean eq = TestHelper.equalJson(sqlResult, sqlppResult, false, false, false, null);
- File sqlResultFile = writeResult(sqlResult, testcaseId, "sql", testcaseDescription);
- File sqlppResultFile = writeResult(sqlppResult, testcaseId, "sqlpp", testcaseDescription);
+ File sqlResultFile = writeResult(outputDir, sqlResult, testcaseId, "sql", testcaseDescription);
+ File sqlppResultFile = writeResult(outputDir, sqlppResult, testcaseId, "sqlpp", testcaseDescription);
if (!eq) {
/*
@@ -329,7 +326,13 @@
}
}
- protected File writeResult(ArrayNode result, int testcaseId, String resultKind, String comment) throws IOException {
+ static File writeResult(Path outputDir, ArrayNode result, int testcaseId, String resultKind, String comment)
+ throws IOException {
+ return writeResult(outputDir, result, testcaseId, resultKind, comment, SqlppRQGTestBase::prettyPrint);
+ }
+
+ static File writeResult(Path outputDir, ArrayNode result, int testcaseId, String resultKind, String comment,
+ JsonNodePrinter printer) throws IOException {
File outDir = outputDir.toFile();
String outFileName = String.format("%d.%s.txt", testcaseId, resultKind);
FileUtils.forceMkdir(outDir);
@@ -338,12 +341,20 @@
pw.print("---");
pw.println(comment);
for (int i = 0, ln = result.size(); i < ln; i++) {
- pw.println(ResultExtractor.prettyPrint(result.get(i)));
+ printer.print(pw, result.get(i));
}
}
return outFile;
}
+ public interface JsonNodePrinter {
+ void print(PrintWriter out, JsonNode node) throws IOException;
+ }
+
+ private static void prettyPrint(PrintWriter out, JsonNode node) throws JsonProcessingException {
+ out.println(ResultExtractor.prettyPrint(node));
+ }
+
protected static <T> List<T> randomize(Collection<T> input, Random random) {
List<T> output = new ArrayList<>(input);
Collections.shuffle(output, random);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate-sql/kurtosis_distinct/kurtosis_distinct.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate-sql/kurtosis_distinct/kurtosis_distinct.1.adm
index 0f3644d..9ad4a6d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate-sql/kurtosis_distinct/kurtosis_distinct.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate-sql/kurtosis_distinct/kurtosis_distinct.1.adm
@@ -1 +1 @@
-{ "t1": -2, "t2": null, "t3": null, "t4": null }
+{ "t1": -1, "t2": null, "t3": null, "t4": null }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate/kurtosis_distinct/kurtosis_distinct.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate/kurtosis_distinct/kurtosis_distinct.1.adm
index 0f3644d..9ad4a6d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate/kurtosis_distinct/kurtosis_distinct.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/aggregate/kurtosis_distinct/kurtosis_distinct.1.adm
@@ -1 +1 @@
-{ "t1": -2, "t2": null, "t3": null, "t4": null }
+{ "t1": -1, "t2": null, "t3": null, "t4": null }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.1.adm
index c471941..24ca59d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.1.adm
@@ -11,11 +11,11 @@
{ "g": 0, "i": 10, "actual": 80 }
{ "g": 0, "i": 11, "actual": 80 }
{ "g": 0, "i": 12, "actual": 80 }
-{ "g": 0, "i": 13, "actual": -81 }
-{ "g": 0, "i": 14, "actual": -81 }
-{ "g": 0, "i": 15, "actual": -81 }
+{ "g": 0, "i": 13, "actual": -80 }
+{ "g": 0, "i": 14, "actual": -80 }
+{ "g": 0, "i": 15, "actual": -80 }
{ "g": 0, "i": 16, "actual": 81 }
-{ "g": 0, "i": 17, "actual": -82 }
+{ "g": 0, "i": 17, "actual": -81 }
{ "g": 0, "i": 18, "actual": 127 }
{ "g": 0, "i": 19, "actual": 127 }
{ "g": 0, "i": 20, "actual": 127 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.2.adm
index 2464aa4..ec0cdff 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.2.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.2.adm
@@ -11,11 +11,11 @@
{ "g": 0, "i": 10, "actual": 160 }
{ "g": 0, "i": 11, "actual": 160 }
{ "g": 0, "i": 12, "actual": 160 }
-{ "g": 0, "i": 13, "actual": -161 }
-{ "g": 0, "i": 14, "actual": -161 }
-{ "g": 0, "i": 15, "actual": -161 }
+{ "g": 0, "i": 13, "actual": -160 }
+{ "g": 0, "i": 14, "actual": -160 }
+{ "g": 0, "i": 15, "actual": -160 }
{ "g": 0, "i": 16, "actual": 161 }
-{ "g": 0, "i": 17, "actual": -162 }
+{ "g": 0, "i": 17, "actual": -161 }
{ "g": 0, "i": 18, "actual": 32767 }
{ "g": 0, "i": 19, "actual": 32767 }
{ "g": 0, "i": 20, "actual": 32767 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.3.adm
index 282733d..46bf8cb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.3.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.3.adm
@@ -11,11 +11,11 @@
{ "g": 0, "i": 10, "actual": 320 }
{ "g": 0, "i": 11, "actual": 320 }
{ "g": 0, "i": 12, "actual": 320 }
-{ "g": 0, "i": 13, "actual": -321 }
-{ "g": 0, "i": 14, "actual": -321 }
-{ "g": 0, "i": 15, "actual": -321 }
+{ "g": 0, "i": 13, "actual": -320 }
+{ "g": 0, "i": 14, "actual": -320 }
+{ "g": 0, "i": 15, "actual": -320 }
{ "g": 0, "i": 16, "actual": 321 }
-{ "g": 0, "i": 17, "actual": -322 }
+{ "g": 0, "i": 17, "actual": -321 }
{ "g": 0, "i": 18, "actual": 2147483647 }
{ "g": 0, "i": 19, "actual": 2147483647 }
{ "g": 0, "i": 20, "actual": 2147483647 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.4.adm
index dccc6fd..b6262aa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.4.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/constructor/int_01/int_01.4.adm
@@ -12,11 +12,11 @@
{ "g": 0, "i": 11, "actual": 640 }
{ "g": 0, "i": 12, "actual": 640 }
{ "g": 0, "i": 13, "actual": 640 }
-{ "g": 0, "i": 14, "actual": -641 }
-{ "g": 0, "i": 15, "actual": -641 }
-{ "g": 0, "i": 16, "actual": -641 }
+{ "g": 0, "i": 14, "actual": -640 }
+{ "g": 0, "i": 15, "actual": -640 }
+{ "g": 0, "i": 16, "actual": -640 }
{ "g": 0, "i": 17, "actual": 641 }
-{ "g": 0, "i": 18, "actual": -642 }
+{ "g": 0, "i": 18, "actual": -641 }
{ "g": 0, "i": 19, "actual": 9223372036854775807 }
{ "g": 0, "i": 20, "actual": 9223372036854775807 }
{ "g": 0, "i": 21, "actual": 9223372036854775807 }
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt16TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt16TypeConvertComputer.java
index 85d71a8..a8e2633 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt16TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt16TypeConvertComputer.java
@@ -85,8 +85,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (short) Math.floor(sourceValue);
+ return (short) sourceValue;
}
private void raiseBoundaryCheckException(double sourceValue) throws HyracksDataException {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt32TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt32TypeConvertComputer.java
index 5653731..4181d41 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt32TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt32TypeConvertComputer.java
@@ -88,8 +88,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (int) Math.floor(sourceValue);
+ return (int) sourceValue;
}
private int raiseBoundaryCheckException(double sourceValue) throws HyracksDataException {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt64TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt64TypeConvertComputer.java
index cb9a6bc..a2bb3a6 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt64TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt64TypeConvertComputer.java
@@ -93,8 +93,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (long) Math.floor(sourceValue);
+ return (long) sourceValue;
}
private void raiseBoundaryCheckException(double sourceValue) throws HyracksDataException {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt8TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt8TypeConvertComputer.java
index 69934b5..afb7ff6 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt8TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/DoubleToInt8TypeConvertComputer.java
@@ -85,8 +85,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (byte) Math.floor(sourceValue);
+ return (byte) sourceValue;
}
private void raiseBoundaryCheckException(double sourceValue) throws HyracksDataException {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt16TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt16TypeConvertComputer.java
index 4906c56..e15dd87 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt16TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt16TypeConvertComputer.java
@@ -85,8 +85,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (short) Math.floor(sourceValue);
+ return (short) sourceValue;
}
private void raiseBoundaryException(float sourceValue) throws HyracksDataException {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt32TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt32TypeConvertComputer.java
index 366278d..bbbc7a8 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt32TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt32TypeConvertComputer.java
@@ -88,8 +88,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (int) Math.floor(sourceValue);
+ return (int) sourceValue;
}
private void raiseBoundaryCheckException(float sourceValue) throws HyracksDataException {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt64TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt64TypeConvertComputer.java
index 291cab6..2416300 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt64TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt64TypeConvertComputer.java
@@ -88,8 +88,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (long) Math.floor(sourceValue);
+ return (long) sourceValue;
}
private void raiseBoundaryCheckException(float sourceValue) throws HyracksDataException {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt8TypeConvertComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt8TypeConvertComputer.java
index 30f19c3..2aff3239 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt8TypeConvertComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/FloatToInt8TypeConvertComputer.java
@@ -84,8 +84,7 @@
}
}
- // Math.floor to truncate decimal portion
- return (byte) Math.floor(sourceValue);
+ return (byte) sourceValue;
}
private void raiseBoundaryException(float sourceValue) throws HyracksDataException {