Merge branch 'cheshire-cat' into 'master'
Change-Id: Ic9ebf963437bf2cac2fb48b43b41728f515133c6
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index ba04967..08e1be4 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -112,14 +112,31 @@
case REQUEST_STATS:
requestStats((ActiveStatsRequestMessage) message);
break;
+ case GENERIC_EVENT:
+ deliverGenericEvent(message);
+ break;
default:
LOGGER.warn("Unknown message type received: " + message.getKind());
}
}
+ private void deliverGenericEvent(ActiveManagerMessage message) throws HyracksDataException {
+ try {
+ ActiveRuntimeId runtimeId = message.getRuntimeId();
+ IActiveRuntime runtime = runtimes.get(runtimeId);
+ if (runtime == null) {
+ LOGGER.warn("Request for a runtime {} that is not registered {}", runtimeId, message);
+ return;
+ }
+ runtime.handleGenericEvent(message);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
private void requestStats(ActiveStatsRequestMessage message) throws HyracksDataException {
try {
- ActiveRuntimeId runtimeId = (ActiveRuntimeId) message.getPayload();
+ ActiveRuntimeId runtimeId = message.getRuntimeId();
IActiveRuntime runtime = runtimes.get(runtimeId);
long reqId = message.getReqId();
if (runtime == null) {
@@ -168,7 +185,7 @@
@SuppressWarnings("squid:S1181") // Catch Error
private void stopRuntime(ActiveManagerMessage message) {
StopRuntimeParameters content = (StopRuntimeParameters) message.getPayload();
- ActiveRuntimeId runtimeId = content.getRuntimeId();
+ ActiveRuntimeId runtimeId = message.getRuntimeId();
IActiveRuntime runtime = runtimes.get(runtimeId);
if (runtime == null) {
LOGGER.warn("Request to stop runtime: " + runtimeId
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
index a52f01e..b8edc64 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
@@ -20,6 +20,7 @@
import java.util.concurrent.TimeUnit;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IActiveRuntime {
@@ -48,4 +49,8 @@
default String getStats() {
return "\"Runtime stats is not available.\"";
}
+
+ default void handleGenericEvent(ActiveManagerMessage event) throws HyracksDataException {
+ throw new IllegalStateException("generic events not supported for runtime " + getRuntimeId());
+ }
}
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
index 1a2af13..4d726cf 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
@@ -21,6 +21,7 @@
import java.io.Serializable;
import org.apache.asterix.active.ActiveManager;
+import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.messaging.CcIdentifiedMessage;
import org.apache.asterix.common.messaging.api.INcAddressedMessage;
@@ -29,15 +30,18 @@
public class ActiveManagerMessage extends CcIdentifiedMessage implements INcAddressedMessage {
public enum Kind {
STOP_ACTIVITY,
- REQUEST_STATS
+ REQUEST_STATS,
+ GENERIC_EVENT
}
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private final Kind kind;
+ private final ActiveRuntimeId runtimeId;
private final Serializable payload;
- public ActiveManagerMessage(Kind kind, Serializable payload) {
+ public ActiveManagerMessage(Kind kind, ActiveRuntimeId runtimeId, Serializable payload) {
this.kind = kind;
+ this.runtimeId = runtimeId;
this.payload = payload;
}
@@ -45,6 +49,10 @@
return payload;
}
+ public ActiveRuntimeId getRuntimeId() {
+ return runtimeId;
+ }
+
public Kind getKind() {
return kind;
}
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java
index 117a68c..94668a0 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java
@@ -18,14 +18,14 @@
*/
package org.apache.asterix.active.message;
-import java.io.Serializable;
+import org.apache.asterix.active.ActiveRuntimeId;
public class ActiveStatsRequestMessage extends ActiveManagerMessage {
private static final long serialVersionUID = 1L;
private final long reqId;
- public ActiveStatsRequestMessage(Serializable payload, long reqId) {
- super(Kind.REQUEST_STATS, payload);
+ public ActiveStatsRequestMessage(ActiveRuntimeId runtimeId, long reqId) {
+ super(Kind.REQUEST_STATS, runtimeId, null);
this.reqId = reqId;
}
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java
index fbc41a1..c21f06e 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java
@@ -21,25 +21,17 @@
import java.io.Serializable;
import java.util.concurrent.TimeUnit;
-import org.apache.asterix.active.ActiveRuntimeId;
-
public class StopRuntimeParameters implements Serializable {
- private static final long serialVersionUID = 1L;
- private final ActiveRuntimeId runtimeId;
+ private static final long serialVersionUID = 2L;
private final long timeout;
private final TimeUnit unit;
- public StopRuntimeParameters(ActiveRuntimeId runtimeId, long timeout, TimeUnit unit) {
- this.runtimeId = runtimeId;
+ public StopRuntimeParameters(long timeout, TimeUnit unit) {
this.timeout = timeout;
this.unit = unit;
}
- public ActiveRuntimeId getRuntimeId() {
- return runtimeId;
- }
-
public long getTimeout() {
return timeout;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 2d82f6f..532fb43 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.algebra.operators.physical;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import org.apache.asterix.common.config.OptimizationConfUtil;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.declared.DataSourceId;
@@ -157,7 +159,7 @@
dataset.getDataverseName(), dataset.getDatasetName(), indexName);
if (secondaryIndex == null) {
throw new AlgebricksException(
- "Code generation error: no index " + indexName + " for dataset " + datasetName);
+ "Code generation error: no index " + indexName + " for " + dataset() + " " + datasetName);
}
IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(unnestMap);
RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index e99817d..0c2eeba 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -85,6 +85,7 @@
import org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
@@ -151,7 +152,7 @@
UnnestingPositionWriterFactory.INSTANCE, null,
new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())),
ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null,
- NoOpWarningCollector.INSTANCE, 0);
+ NoOpWarningCollector.INSTANCE, 0, new PhysicalOptimizationConfig());
}
@Override
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
index b6dbbe7..aa2b8fd 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.optimizer.rules;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -254,7 +256,7 @@
// the user query provides zero parameter for the meta function.
if (variableRequired) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, expr.getSourceLocation(),
- "Cannot resolve ambiguous meta function call. There are more than one dataset choice!");
+ "Cannot resolve ambiguous meta function call. There are more than one " + dataset() + " choice");
}
VariableReferenceExpression metaVarRef = new VariableReferenceExpression(metaVar);
metaVarRef.setSourceLocation(expr.getSourceLocation());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
index d466446..5ec1a7f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.optimizer.rules;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -265,7 +267,8 @@
INodeDomain storageDomain = mp.findNodeDomain(dataset.getNodeGroupName());
if (dsi == null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, op.getSourceLocation(),
- "Could not find index " + jobGenParams.getIndexName() + " for dataset " + dataSourceId);
+ "Could not find index " + jobGenParams.getIndexName() + " for " + dataset() + " "
+ + dataSourceId);
}
IndexType indexType = jobGenParams.getIndexType();
boolean requiresBroadcast = jobGenParams.getRequiresBroadcast();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 9bfa8fb..2509010 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -125,9 +125,6 @@
@Override
public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
- for (Mutable<ILogicalExpression> me : op.getExpressions()) {
- sweepExpression(me.getValue());
- }
return null;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 02dc6f6..1b7b7d2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -28,6 +28,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.function.Predicate;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -47,6 +48,7 @@
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
@@ -99,10 +101,11 @@
* The visitor ensures that the variables used to correlate between the
* query plan rooted at <code>subplanInputOperator</code> are propagated
* to the operator being visited.
- *
- * ----------------------------------
+ * <p>
* Here is an abstract example.
+ * <p>
* The original query plan:
+ * <pre>
* --Op1
* --Subplan{
* --AggregateOp
@@ -112,9 +115,10 @@
* }
* --InputOp
* .....
- *
+ * </pre>
* After we call NestedOp.accept(....) with this visitor. We will get an
* intermediate plan that looks like:
+ * <pre>
* --Op1
* --Subplan{
* --AggregateOp
@@ -125,9 +129,9 @@
* }
* --InputOp
* .....
+ * </pre>
* The plan rooted at InputOp' is a deep copy of the plan rooted at InputOp
* with a different set of variables.
- *
*/
class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOperator, Void> {
// The optimization context.
@@ -186,17 +190,6 @@
}
@Override
- public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
- return visitAggregateOperator(op);
- }
-
- @Override
- public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
- throws AlgebricksException {
- return visitAggregateOperator(op);
- }
-
- @Override
public ILogicalOperator visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg)
throws AlgebricksException {
return visitSingleInputOperator(op);
@@ -248,14 +241,38 @@
}
@Override
+ public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
+ throws AlgebricksException {
+ return visitLimitOrRunningAggregateOperator(op, InlineAllNtsInSubplanVisitor::isCreateQueryUidRunningAggregate);
+ }
+
+ /*
+ * create-query-uid() doesn't require GROUP BY wrapping in the presence of correlated keys
+ */
+ private static boolean isCreateQueryUidRunningAggregate(RunningAggregateOperator ragg) {
+ return ragg.getExpressions().stream().map(Mutable::getValue)
+ .allMatch(expr -> expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL
+ && BuiltinFunctions.CREATE_QUERY_UID
+ .equals(((AbstractFunctionCallExpression) expr).getFunctionIdentifier()));
+ }
+
+ @Override
public ILogicalOperator visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+ return visitLimitOrRunningAggregateOperator(op, null);
+ }
+
+ private <T extends ILogicalOperator> ILogicalOperator visitLimitOrRunningAggregateOperator(T op,
+ Predicate<T> noCorrelatedKeysHandlingTest) throws AlgebricksException {
// Processes its input operator.
visitSingleInputOperator(op);
if (correlatedKeyVars.isEmpty()) {
return op;
}
+ if (noCorrelatedKeysHandlingTest != null && noCorrelatedKeysHandlingTest.test(op)) {
+ return op;
+ }
- // Get live variables before limit.
+ // Get live variables before limit (or running aggregate) operator.
Set<LogicalVariable> inputLiveVars = new HashSet<LogicalVariable>();
VariableUtilities.getSubplanLocalLiveVariables(op.getInputs().get(0).getValue(), inputLiveVars);
@@ -267,8 +284,9 @@
ILogicalOperator inputOp = op.getInputs().get(0).getValue();
assignOp.getInputs().add(new MutableObject<>(inputOp));
- // Rewrites limit to a group-by with limit as its nested operator.
- Pair<ILogicalOperator, LogicalVariable> gbyOpAndAggVar = wrapLimitInGroupBy(op, recordVar, inputLiveVars);
+ // Rewrites limit (or running aggregate) to a group-by with limit (or running aggregate) as its nested operator.
+ Pair<ILogicalOperator, LogicalVariable> gbyOpAndAggVar =
+ wrapLimitOrRunningAggregateInGroupBy(op, recordVar, inputLiveVars);
ILogicalOperator gbyOp = gbyOpAndAggVar.first;
LogicalVariable aggVar = gbyOpAndAggVar.second;
gbyOp.getInputs().add(new MutableObject<>(assignOp));
@@ -312,8 +330,8 @@
return new Pair<>(assignOp, recordVar);
}
- private Pair<ILogicalOperator, LogicalVariable> wrapLimitInGroupBy(ILogicalOperator op, LogicalVariable recordVar,
- Set<LogicalVariable> inputLiveVars) throws AlgebricksException {
+ private Pair<ILogicalOperator, LogicalVariable> wrapLimitOrRunningAggregateInGroupBy(ILogicalOperator op,
+ LogicalVariable recordVar, Set<LogicalVariable> inputLiveVars) throws AlgebricksException {
SourceLocation sourceLoc = op.getSourceLocation();
GroupByOperator gbyOp = new GroupByOperator();
gbyOp.setSourceLocation(sourceLoc);
@@ -348,7 +366,7 @@
AggregateOperator aggOp = new AggregateOperator(aggVarList, aggExprList);
aggOp.setSourceLocation(sourceLoc);
- // Adds the original limit operator as the input operator to the added
+ // Adds the original limit (or running aggregate) operator as the input operator to the added
// aggregate operator.
aggOp.getInputs().add(new MutableObject<>(op));
op.getInputs().clear();
@@ -361,7 +379,7 @@
}
// Adds a nested tuple source operator as the input operator to the
- // limit operator.
+ // limit (or running aggregate) operator.
NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<>(gbyOp));
nts.setSourceLocation(sourceLoc);
currentOp.getInputs().add(new MutableObject<>(nts));
@@ -450,10 +468,11 @@
// Updates the primary key info in the copied plan segment.
Map<LogicalVariable, LogicalVariable> varMap = deepCopyVisitor.getInputToOutputVariableMapping();
addPrimaryKeys(varMap);
- Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars =
+ EquivalenceClassUtils.computePrimaryKeys(copiedInputOperator, context);
+ Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> primaryOpAndVars =
EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(copiedInputOperator, true, context);
correlatedKeyVars.clear();
- correlatedKeyVars.addAll(primaryOpAndVars.second);
+ correlatedKeyVars.addAll(primaryOpAndVars.first);
// Update key variables and input-output-var mapping.
varMap.forEach((oldVar, newVar) -> {
if (correlatedKeyVars.contains(oldVar)) {
@@ -462,7 +481,14 @@
}
updateInputToOutputVarMapping(oldVar, newVar, true);
});
- return primaryOpAndVars.first;
+ if (primaryOpAndVars.second == null) {
+ // found primary key for the existing operator (copiedInputOperator)
+ return copiedInputOperator;
+ } else {
+ ILogicalOperator newInputOp = primaryOpAndVars.second;
+ context.addPrimaryKey(primaryOpAndVars.third);
+ return newInputOp;
+ }
}
@Override
@@ -669,19 +695,20 @@
}
/**
- * Wraps an AggregateOperator or RunningAggregateOperator with a group-by
+ * Wraps an AggregateOperator with a group-by
* operator where the group-by keys are variables in keyVarsToEnforce. Note
* that the function here prevents this visitor being used to rewrite
* arbitrary query plans. Instead, it could only be used for rewriting a
* nested plan within a subplan operator.
*
* @param op
- * the logical operator for aggregate or running aggregate.
+ * the logical operator for aggregate.
* @return the wrapped group-by operator if {@code keyVarsToEnforce} is not
* empty, and {@code op} otherwise.
* @throws AlgebricksException
*/
- private ILogicalOperator visitAggregateOperator(ILogicalOperator op) throws AlgebricksException {
+ @Override
+ public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
visitSingleInputOperator(op);
if (correlatedKeyVars.isEmpty()) {
return op;
@@ -808,5 +835,4 @@
new FunctionalDependency(newDependencies, Collections.singletonList(entry.getValue())));
}
}
-
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
index 81b4a3d..ae9a886 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
@@ -26,6 +26,8 @@
import java.util.Map;
import java.util.Set;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
@@ -33,6 +35,7 @@
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -54,6 +57,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -365,27 +369,35 @@
Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
ILogicalOperator inputOpBackup = inputOpRef.getValue();
// Creates parameters for the left outer join operator.
- Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars =
+ EquivalenceClassUtils.computePrimaryKeys(inputOpBackup, context);
+ Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> primaryOpAndVars =
EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, true, context);
- ILogicalOperator inputOp = primaryOpAndVars.first;
- Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
- inputOpRef.setValue(inputOp);
+ Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.first;
+ FunctionalDependency newPrimaryKeyFd = null;
+ ILogicalOperator inputOp;
+ if (primaryOpAndVars.second != null) {
+ // there's a new operator that generates a primary key
+ inputOp = primaryOpAndVars.second;
+ inputOpRef.setValue(inputOp);
+ newPrimaryKeyFd = primaryOpAndVars.third;
+ } else {
+ inputOp = inputOpBackup;
+ }
Set<LogicalVariable> inputLiveVars = new HashSet<>();
VariableUtilities.getLiveVariables(inputOp, inputLiveVars);
Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> varMapAndOrderExprs =
- SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context);
+ SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context, newPrimaryKeyFd);
Map<LogicalVariable, LogicalVariable> varMap = varMapAndOrderExprs.first;
if (varMap == null) {
inputOpRef.setValue(inputOpBackup);
return new Pair<>(false, new LinkedHashMap<>());
}
-
Mutable<ILogicalOperator> lowestAggregateRefInSubplan =
SubplanFlatteningUtil.findLowestAggregate(subplanOp.getNestedPlans().get(0).getRoots().get(0));
if (lowestAggregateRefInSubplan == null) {
- inputOpRef.setValue(inputOpBackup);
- return new Pair<>(false, new LinkedHashMap<>());
+ // not supposed to happen. SubplanFlatteningUtil.inlineAllNestedTupleSource() checks for it
+ throw new CompilationException(ErrorCode.ILLEGAL_STATE, subplanOp.getSourceLocation());
}
Mutable<ILogicalOperator> rightInputOpRef = lowestAggregateRefInSubplan.getValue().getInputs().get(0);
@@ -525,21 +537,29 @@
ILogicalOperator inputOpBackup = inputOpRef.getValue();
// Gets live variables and covering variables from the subplan's input operator.
- Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars =
+ EquivalenceClassUtils.computePrimaryKeys(inputOpBackup, context);
+ Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> primaryOpAndVars =
EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, false, context);
- ILogicalOperator inputOp = primaryOpAndVars.first;
- Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
- inputOpRef.setValue(inputOp);
+ Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.first;
+ FunctionalDependency newPrimaryKeyFd = null;
+ ILogicalOperator inputOp;
+ if (primaryOpAndVars.second != null) {
+ // there's a new operator that generates a primary key
+ inputOp = primaryOpAndVars.second;
+ inputOpRef.setValue(inputOp);
+ newPrimaryKeyFd = primaryOpAndVars.third;
+ } else {
+ inputOp = inputOpBackup;
+ }
Set<LogicalVariable> liveVars = new HashSet<>();
VariableUtilities.getLiveVariables(inputOp, liveVars);
Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>> notNullVarsAndTopJoinRef =
- SubplanFlatteningUtil.inlineLeftNtsInSubplanJoin(subplanOp, context);
+ SubplanFlatteningUtil.inlineLeftNtsInSubplanJoin(subplanOp, context, newPrimaryKeyFd);
if (notNullVarsAndTopJoinRef.first == null) {
inputOpRef.setValue(inputOpBackup);
return new Pair<>(false, replacedVarMap);
}
-
Set<LogicalVariable> notNullVars = notNullVarsAndTopJoinRef.first;
Mutable<ILogicalOperator> topJoinRef = notNullVarsAndTopJoinRef.second;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java
index 09377fa..e3c7d69 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java
@@ -35,28 +35,35 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import com.google.common.collect.ImmutableSet;
-
class SubplanFlatteningUtil {
+ private static final Set<LogicalOperatorTag> OP_SET_NESTEDTUPLESOURCE =
+ EnumSet.of(LogicalOperatorTag.NESTEDTUPLESOURCE);
+
+ private static final Set<LogicalOperatorTag> OP_SET_INNER_OUTER_JOIN =
+ EnumSet.of(LogicalOperatorTag.INNERJOIN, LogicalOperatorTag.LEFTOUTERJOIN);
+
/**
* Blindly inline all NTS's in a Subplan operator.
*
* @param subplanOp,
* the subplan operator
* @param context
+ * the optimization context
+ * @param extraPrimaryKeyFd
+ * extra primary key dependency that needs to be added to the context before performing the rewrite
* @return a map that maps primary key variables in the subplan's input to its deep copies
* in the nested pipeline; the ordering that needs to be maintained for the final
* aggregation in the added group-by operator.
- * @throws AlgebricksException
*/
public static Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> inlineAllNestedTupleSource(
- SubplanOperator subplanOp, IOptimizationContext context) throws AlgebricksException {
+ SubplanOperator subplanOp, IOptimizationContext context, FunctionalDependency extraPrimaryKeyFd)
+ throws AlgebricksException {
// For nested subplan, we do not continue for the general inlining.
- if (OperatorManipulationUtil.ancestorOfOperators(subplanOp,
- ImmutableSet.of(LogicalOperatorTag.NESTEDTUPLESOURCE))) {
+ if (OperatorManipulationUtil.ancestorOfOperators(subplanOp, OP_SET_NESTEDTUPLESOURCE)) {
return new Pair<>(null, null);
}
@@ -65,6 +72,10 @@
return new Pair<>(null, null);
}
+ if (extraPrimaryKeyFd != null) {
+ context.addPrimaryKey(extraPrimaryKeyFd);
+ }
+
// Rewrites the query plan.
InlineAllNtsInSubplanVisitor visitor = new InlineAllNtsInSubplanVisitor(context, subplanOp);
ILogicalOperator topOp = topOpRef.getValue();
@@ -77,8 +88,7 @@
// Gets ordering variables.
List<Pair<IOrder, Mutable<ILogicalExpression>>> orderVars = visitor.getOrderingExpressions();
- return new Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>>(
- visitor.getInputVariableToOutputVariableMap(), orderVars);
+ return new Pair<>(visitor.getInputVariableToOutputVariableMap(), orderVars);
}
/**
@@ -89,17 +99,23 @@
* the SubplanOperator
* @param context
* the optimization context
+ * @param extraPrimaryKeyFd
+ * extra primary key dependency that needs to be added to the context before performing the rewrite
* @return A set of variables used for further null-checks, i.e., variables indicating
* whether a tuple produced by a transformed left outer join is a non-match;
* a reference to the top join operator in the nested subplan.
- * @throws AlgebricksException
*/
public static Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>> inlineLeftNtsInSubplanJoin(
- SubplanOperator subplanOp, IOptimizationContext context) throws AlgebricksException {
+ SubplanOperator subplanOp, IOptimizationContext context, FunctionalDependency extraPrimaryKeyFd)
+ throws AlgebricksException {
Pair<Boolean, ILogicalOperator> applicableAndNtsToRewrite =
SubplanFlatteningUtil.isQualifiedForSpecialFlattening(subplanOp);
if (!applicableAndNtsToRewrite.first) {
- return new Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>>(null, null);
+ return new Pair<>(null, null);
+ }
+
+ if (extraPrimaryKeyFd != null) {
+ context.addPrimaryKey(extraPrimaryKeyFd);
}
ILogicalOperator qualifiedNts = applicableAndNtsToRewrite.second;
@@ -126,8 +142,7 @@
VariableUtilities.substituteVariables(currentOp, subplanLocalVarMap, context);
currentOp = currentOp.getInputs().get(0).getValue();
}
- return new Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>>(specialVisitor.getNullCheckVariables(),
- topJoinRef);
+ return new Pair<>(specialVisitor.getNullCheckVariables(), topJoinRef);
}
/**
@@ -206,25 +221,23 @@
* @param subplanOp,
* the SubplanOperator to consider
* @return TRUE if the rewriting is applicable; FALSE otherwise.
- * @throws AlgebricksException
*/
private static Pair<Boolean, ILogicalOperator> isQualifiedForSpecialFlattening(SubplanOperator subplanOp)
throws AlgebricksException {
if (!OperatorManipulationUtil.ancestorOfOperators(
subplanOp.getNestedPlans().get(0).getRoots().get(0).getValue(),
// we don't need to check recursively for this special rewriting.
- EnumSet.of(LogicalOperatorTag.INNERJOIN, LogicalOperatorTag.LEFTOUTERJOIN))) {
- return new Pair<Boolean, ILogicalOperator>(false, null);
+ OP_SET_INNER_OUTER_JOIN)) {
+ return new Pair<>(false, null);
}
SubplanSpecialFlatteningCheckVisitor visitor = new SubplanSpecialFlatteningCheckVisitor();
for (ILogicalPlan plan : subplanOp.getNestedPlans()) {
for (Mutable<ILogicalOperator> opRef : plan.getRoots()) {
if (!opRef.getValue().accept(visitor, null)) {
- return new Pair<Boolean, ILogicalOperator>(false, null);
+ return new Pair<>(false, null);
}
}
}
- return new Pair<Boolean, ILogicalOperator>(true, visitor.getQualifiedNts());
+ return new Pair<>(true, visitor.getQualifiedNts());
}
-
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
index 1593cf1..24f368f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
@@ -22,40 +22,49 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.WindowOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.PrimaryKeyVariablesVisitor;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
public class EquivalenceClassUtils {
+ // Controls whether to use create-query-uid() function when generating a primary key
+ // If disabled then use row_number() approach instead
+ public static final String REWRITE_INTERNAL_QUERYUID_PK = "rewrite_internal_queryuid_pk";
+ static final boolean REWRITE_INTERNAL_QUERYUID_PK_DEFAULT = true;
+
/**
* Adds equivalent classes for primary index accesses, including unnest-map for
* primary index access and data source scan through primary index ---
@@ -148,16 +157,14 @@
* a set of primary key variables at the operator.
* @throws AlgebricksException
*/
- public static Pair<ILogicalOperator, Set<LogicalVariable>> findOrCreatePrimaryKeyOpAndVariables(
+ public static Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> findOrCreatePrimaryKeyOpAndVariables(
ILogicalOperator operator, boolean usedForCorrelationJoin, IOptimizationContext context)
throws AlgebricksException {
- computePrimaryKeys(operator, context);
-
- Set<LogicalVariable> liveVars = new HashSet<>();
+ Set<LogicalVariable> liveVars = new LinkedHashSet<>();
VariableUtilities.getSubplanLocalLiveVariables(operator, liveVars);
- Set<LogicalVariable> primaryKeyVars = new HashSet<>();
- Set<LogicalVariable> noKeyVars = new HashSet<>();
+ Set<LogicalVariable> primaryKeyVars = new LinkedHashSet<>();
+ Set<LogicalVariable> noKeyVars = new LinkedHashSet<>();
for (LogicalVariable liveVar : liveVars) {
List<LogicalVariable> keyVars = context.findPrimaryKey(liveVar);
if (keyVars != null) {
@@ -171,24 +178,57 @@
}
primaryKeyVars.retainAll(liveVars);
if (primaryKeyVars.containsAll(noKeyVars)) {
- return new Pair<ILogicalOperator, Set<LogicalVariable>>(operator, primaryKeyVars);
+ return new Triple<>(primaryKeyVars, null, null);
+ } else if (!usedForCorrelationJoin && isQueryUidPkEnabled(context)) {
+ LogicalVariable idVar = context.newVar();
+ RunningAggregateOperator assignIdOp =
+ new RunningAggregateOperator(idVar, new MutableObject<>(new StatefulFunctionCallExpression(
+ BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.CREATE_QUERY_UID), null)));
+ assignIdOp.setSourceLocation(operator.getSourceLocation());
+ assignIdOp.getInputs().add(new MutableObject<>(operator));
+
+ context.computeAndSetTypeEnvironmentForOperator(assignIdOp);
+
+ FunctionalDependency primaryKeyFD =
+ new FunctionalDependency(Collections.singletonList(idVar), new ArrayList<>(liveVars));
+
+ return new Triple<>(Collections.singleton(idVar), assignIdOp, primaryKeyFD);
} else {
- LogicalVariable assignVar = context.newVar();
- ILogicalOperator assignOp = new AssignOperator(assignVar,
- new MutableObject<ILogicalExpression>(new StatefulFunctionCallExpression(
- FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_QUERY_UID), null)));
- OperatorPropertiesUtil.markMovable(assignOp, !usedForCorrelationJoin);
- assignOp.getInputs().add(new MutableObject<ILogicalOperator>(operator));
- context.addPrimaryKey(new FunctionalDependency(Collections.singletonList(assignVar),
- new ArrayList<LogicalVariable>(liveVars)));
- context.computeAndSetTypeEnvironmentForOperator(assignOp);
- return new Pair<ILogicalOperator, Set<LogicalVariable>>(assignOp, Collections.singleton(assignVar));
+ noKeyVars.removeAll(primaryKeyVars);
+
+ List<Mutable<ILogicalExpression>> partitionVarRefs =
+ new ArrayList<>(primaryKeyVars.size() + noKeyVars.size());
+ OperatorManipulationUtil.createVariableReferences(primaryKeyVars, operator.getSourceLocation(),
+ partitionVarRefs);
+ OperatorManipulationUtil.createVariableReferences(noKeyVars, operator.getSourceLocation(),
+ partitionVarRefs);
+ LogicalVariable rowNumVar = context.newVar();
+ AbstractFunctionCallExpression rowNumExpr = BuiltinFunctions
+ .makeWindowFunctionExpression(BuiltinFunctions.ROW_NUMBER_IMPL, Collections.emptyList());
+ WindowOperator winOp = new WindowOperator(partitionVarRefs, Collections.emptyList());
+ winOp.getVariables().add(rowNumVar);
+ winOp.getExpressions().add(new MutableObject<>(rowNumExpr));
+ winOp.setSourceLocation(operator.getSourceLocation());
+ winOp.getInputs().add(new MutableObject<>(operator));
+
+ context.computeAndSetTypeEnvironmentForOperator(winOp);
+
+ primaryKeyVars.addAll(noKeyVars);
+ primaryKeyVars.add(rowNumVar);
+ FunctionalDependency primaryKeyFD =
+ new FunctionalDependency(new ArrayList<>(primaryKeyVars), new ArrayList<>(liveVars));
+
+ return new Triple<>(primaryKeyVars, winOp, primaryKeyFD);
}
}
- private static void computePrimaryKeys(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ public static void computePrimaryKeys(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
PrimaryKeyVariablesVisitor visitor = new PrimaryKeyVariablesVisitor();
PhysicalOptimizationsUtil.visitOperatorAndItsDescendants(op, visitor, ctx);
}
+ private static boolean isQueryUidPkEnabled(IOptimizationContext context) {
+ MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+ return metadataProvider.getBooleanProperty(REWRITE_INTERNAL_QUERYUID_PK, REWRITE_INTERNAL_QUERYUID_PK_DEFAULT);
+ }
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
index 58bdaca..6f07718 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.translator;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
@@ -57,6 +60,7 @@
import org.apache.asterix.metadata.dataset.hints.DatasetHints;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.utils.MetadataConstants;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -73,11 +77,12 @@
protected static final String INVALID_OPERATION_MESSAGE = "Invalid operation - %s";
- protected static final String BAD_DATAVERSE_DML_MESSAGE = "%s operation is not permitted in dataverse %s";
+ protected static final String BAD_DATAVERSE_DML_MESSAGE = "%s operation is not permitted in " + dataverse() + " %s";
- protected static final String BAD_DATAVERSE_DDL_MESSAGE = "Cannot %s dataverse: %s";
+ protected static final String BAD_DATAVERSE_DDL_MESSAGE = "Cannot %s " + dataverse() + ": %s";
- protected static final String BAD_DATAVERSE_OBJECT_DDL_MESSAGE = "Cannot %s a %s belonging to the dataverse: %s";
+ protected static final String BAD_DATAVERSE_OBJECT_DDL_MESSAGE =
+ "Cannot %s a %s belonging to the " + dataverse() + ": %s";
public void validateOperation(ICcApplicationContext appCtx, Dataverse defaultDataverse, Statement stmt)
throws AlgebricksException {
@@ -210,7 +215,7 @@
}
invalidOperation = isMetadataDataverse(dataverseName);
if (invalidOperation) {
- message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "create", "dataset", dataverseName);
+ message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "create", dataset(), dataverseName);
}
if (!invalidOperation) {
@@ -221,9 +226,9 @@
Pair<Boolean, String> validationResult =
DatasetHints.validate(appCtx, hint.getKey(), hint.getValue());
if (!validationResult.first) {
- errorMsgBuffer.append("Dataset: ").append(dsCreateStmt.getName().getValue())
- .append(" error in processing hint: ").append(hint.getKey()).append(" ")
- .append(validationResult.second);
+ errorMsgBuffer.append(StringUtils.capitalize(dataset())).append(": ")
+ .append(dsCreateStmt.getName().getValue()).append(" error in processing hint: ")
+ .append(hint.getKey()).append(" ").append(validationResult.second);
errorMsgBuffer.append(" \n");
}
}
@@ -242,7 +247,7 @@
}
invalidOperation = isMetadataDataverse(dataverseName);
if (invalidOperation) {
- message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "drop", "dataset", dataverseName);
+ message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "drop", dataset(), dataverseName);
}
break;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index fd689a5..da07acb 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.translator;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -206,8 +208,8 @@
validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName(), sourceLoc);
List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
if (dataset.hasMetaPart()) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- dataset.getDatasetName() + ": load dataset is not supported on Datasets with Meta records");
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, dataset.getDatasetName() + ": load "
+ + dataset() + " is not supported on " + dataset() + "s with meta records");
}
LoadableDataSource lds;
@@ -430,8 +432,8 @@
SourceLocation sourceLoc = stmt.getSourceLocation();
if (targetDatasource.getDataset().hasMetaPart()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- targetDatasource.getDataset().getDatasetName()
- + ": delete from dataset is not supported on Datasets with Meta records");
+ targetDatasource.getDataset().getDatasetName() + ": delete from " + dataset()
+ + " is not supported on " + dataset() + "s with meta records");
}
List<String> filterField = DatasetUtil.getFilterField(targetDatasource.getDataset());
@@ -461,8 +463,8 @@
SourceLocation sourceLoc = stmt.getSourceLocation();
if (!targetDatasource.getDataset().allow(topOp, DatasetUtil.OP_UPSERT)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- targetDatasource.getDataset().getDatasetName()
- + ": upsert into dataset is not supported on Datasets with Meta records");
+ targetDatasource.getDataset().getDatasetName() + ": upsert into " + dataset()
+ + " is not supported on " + dataset() + "s with meta records");
}
ProjectOperator project = (ProjectOperator) topOp;
CompiledUpsertStatement compiledUpsert = (CompiledUpsertStatement) stmt;
@@ -474,7 +476,7 @@
if (targetDatasource.getDataset().hasMetaPart()) {
if (returnExpression != null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Returning not allowed on datasets with Meta records");
+ "Returning not allowed on " + dataset() + "s with meta records");
}
List<LogicalVariable> metaAndKeysVars;
List<Mutable<ILogicalExpression>> metaAndKeysExprs;
@@ -585,8 +587,8 @@
SourceLocation sourceLoc = stmt.getSourceLocation();
if (targetDatasource.getDataset().hasMetaPart()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- targetDatasource.getDataset().getDatasetName()
- + ": insert into dataset is not supported on Datasets with Meta records");
+ targetDatasource.getDataset().getDatasetName() + ": insert into " + dataset()
+ + " is not supported on " + dataset() + "s with meta records");
}
List<String> filterField = DatasetUtil.getFilterField(targetDatasource.getDataset());
@@ -683,7 +685,7 @@
}
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Cannot write output to an external dataset.");
+ "Cannot write output to an external " + dataset());
}
DataSourceId sourceId = new DataSourceId(dataverseName, datasetName);
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index b83fbdf..ccd38d3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -73,6 +73,7 @@
import org.apache.asterix.optimizer.base.FuzzyUtils;
import org.apache.asterix.optimizer.rules.DisjunctivePredicateToJoinRule;
import org.apache.asterix.optimizer.rules.SetAsterixPhysicalOperatorsRule;
+import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.ExecutionPlans;
@@ -134,7 +135,8 @@
CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
+ CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, CompilerProperties.COMPILER_INDEXONLY_KEY,
+ CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, CompilerProperties.COMPILER_ARRAYINDEX_KEY,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
@@ -143,7 +145,8 @@
FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
- SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN);
+ SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN,
+ EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK);
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
index 2fafcfc..353b849 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
@@ -19,6 +19,8 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import java.io.IOException;
import java.io.PrintWriter;
@@ -34,6 +36,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.utils.FlushDatasetUtil;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.io.FileSplit;
@@ -98,8 +101,8 @@
metadataProvider.setMetadataTxnContext(mdTxnCtx);
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
- jsonResponse.put("error",
- "Dataset " + datasetName + " does not exist in " + "dataverse " + dataverseName);
+ jsonResponse.put("error", StringUtils.capitalize(dataset()) + " " + datasetName
+ + " does not exist in " + dataverse() + " " + dataverseName);
out.write(jsonResponse.toString());
out.flush();
return;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index f164938..717ebf8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -83,6 +83,7 @@
protected Path workingDir;
protected String sysAuthHeader;
private ILibraryManager libraryManager;
+ private int timeout;
private static final Logger LOGGER = LogManager.getLogger();
@@ -91,6 +92,7 @@
super(ctx, paths, appCtx, compilationProvider, httpServerProtocol, httpServerPort);
this.compilationProvider = compilationProvider;
this.receptionist = appCtx.getReceptionist();
+ this.timeout = appCtx.getExternalProperties().getLibraryDeployTimeout();
}
@Override
@@ -152,7 +154,7 @@
InternalRequestResponse responseMsg;
try {
ncMb.sendMessageToPrimaryCC(requestMessage);
- responseMsg = (InternalRequestResponse) responseFuture.get(120000, TimeUnit.MILLISECONDS);
+ responseMsg = (InternalRequestResponse) responseFuture.get(timeout, TimeUnit.SECONDS);
} finally {
ncMb.deregisterMessageFuture(responseFuture.getFutureId());
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 5153ebb..a043050 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -280,10 +280,12 @@
final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads());
buildResponseHeaders(requestRef, sessionOutput, param, responsePrinter, delivery);
responsePrinter.printHeaders();
- validateStatement(param.getStatement());
- String statementsText = param.getStatement() + ";";
+ String statement = param.getStatement();
+ statement = statement == null || (!statement.isEmpty() && statement.charAt(statement.length() - 1) == ';')
+ ? statement : (statement + ";");
+ validateStatement(statement);
if (param.isParseOnly()) {
- ResultUtil.ParseOnlyResult parseOnlyResult = parseStatement(statementsText);
+ ResultUtil.ParseOnlyResult parseOnlyResult = parseStatement(statement);
setAccessControlHeaders(request, response);
executionState.setStatus(ResultStatus.SUCCESS, HttpResponseStatus.OK);
response.setStatus(executionState.getHttpStatus());
@@ -296,9 +298,9 @@
IStatementExecutor.StatementProperties statementProperties =
new IStatementExecutor.StatementProperties();
response.setStatus(HttpResponseStatus.OK);
- executeStatement(request, requestRef, statementsText, sessionOutput, resultProperties,
- statementProperties, stats, param, executionState, param.getOptionalParams(), statementParams,
- responsePrinter, warnings);
+ executeStatement(request, requestRef, statement, sessionOutput, resultProperties, statementProperties,
+ stats, param, executionState, param.getOptionalParams(), statementParams, responsePrinter,
+ warnings);
executionState.setStatus(ResultStatus.SUCCESS, HttpResponseStatus.OK);
}
errorCount = 0;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
index 8b04cc3..c4d664b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
@@ -19,6 +19,7 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
@@ -121,13 +122,13 @@
// If a user gives parameter datasetName, she should give dataverseName as well.
if (dataverseName == null && datasetName != null) {
sendResponse(response, HttpResponseStatus.BAD_REQUEST,
- "to rebalance a particular dataset, the parameter dataverseName must be given");
+ "to rebalance a particular " + dataset() + ", the parameter dataverseName must be given");
return;
}
// Does not allow rebalancing a metadata dataset.
if (MetadataConstants.METADATA_DATAVERSE_NAME.equals(dataverseName)) {
- sendResponse(response, HttpResponseStatus.BAD_REQUEST, "cannot rebalance a metadata dataset");
+ sendResponse(response, HttpResponseStatus.BAD_REQUEST, "cannot rebalance a metadata " + dataset());
return;
}
// Schedules a rebalance task and wait for its completion.
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index c90cde0..5f7d65e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -515,7 +515,7 @@
}
ActiveRuntimeId runtimeId = getActiveRuntimeId(partition++);
messageBroker.sendApplicationMessageToNC(new ActiveManagerMessage(ActiveManagerMessage.Kind.STOP_ACTIVITY,
- new StopRuntimeParameters(runtimeId, timeout, unit)), location);
+ runtimeId, new StopRuntimeParameters(timeout, unit)), location);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
index 974cd9e..0d20d51 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -67,7 +69,7 @@
if (unnest.getPositionalVariable() != null) {
// TODO remove this after enabling the support of positional variables in data scan
throw new CompilationException(ErrorCode.COMPILATION_ERROR, unnest.getSourceLocation(),
- "No positional variables are allowed over datasets.");
+ "No positional variables are allowed over " + dataset() + "s");
}
MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
@@ -122,7 +124,7 @@
IAType type = metadata.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
if (type == null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, datasetFnCall.getSourceLocation(),
- "No type for dataset " + dataset.getDatasetName());
+ "No type for " + dataset() + " " + dataset.getDatasetName());
}
return type;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java
index 90374c3..b01ea65 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -129,7 +131,8 @@
if (dataset.hasMetaPart()) {
String metaTypeName = FeedUtils.getFeedMetaTypeName(sourceFeed.getConfiguration());
if (metaTypeName == null) {
- throw new AlgebricksException("Feed to a dataset with metadata doesn't have meta type specified");
+ throw new AlgebricksException(
+ "Feed to a " + dataset() + " with metadata doesn't have meta type specified");
}
metaType = (ARecordType) metadataProvider.findType(id.getDataverseName(), metaTypeName);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
new file mode 100644
index 0000000..5e52517
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import static org.apache.hyracks.util.ExitUtil.EC_NC_FAILED_TO_NOTIFY_TASKS_COMPLETED;
+
+import java.util.Map;
+
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.dataflow.DatasetLocalResource;
+import org.apache.asterix.common.messaging.CcIdentifiedMessage;
+import org.apache.asterix.common.messaging.api.INCMessageBroker;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
+import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
+import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.IIndex;
+import org.apache.hyracks.storage.common.LocalResource;
+import org.apache.hyracks.util.ExitUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
+
+public class StorageCleanupRequestMessage extends CcIdentifiedMessage implements INcAddressedMessage {
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOGGER = LogManager.getLogger();
+ private final IntOpenHashSet validDatasetIds;
+ private final long reqId;
+
+ public StorageCleanupRequestMessage(long reqId, IntOpenHashSet validDatasetIds) {
+ this.validDatasetIds = validDatasetIds;
+ this.reqId = reqId;
+ }
+
+ @Override
+ public void handle(INcApplicationContext appContext) throws HyracksDataException, InterruptedException {
+ INCMessageBroker broker = (INCMessageBroker) appContext.getServiceContext().getMessageBroker();
+ PersistentLocalResourceRepository localResourceRepository =
+ (PersistentLocalResourceRepository) appContext.getLocalResourceRepository();
+ Map<Long, LocalResource> localResources = localResourceRepository.loadAndGetAllResources();
+ for (LocalResource resource : localResources.values()) {
+ DatasetLocalResource lr = (DatasetLocalResource) resource.getResource();
+ if (MetadataIndexImmutableProperties.isMetadataDataset(lr.getDatasetId())) {
+ // skip metadata indexes
+ continue;
+ }
+ if (!validDatasetIds.contains(lr.getDatasetId())) {
+ LOGGER.warn("found invalid index {} with dataset id {}", resource.getPath(), lr.getDatasetId());
+ deleteInvalidIndex(appContext, localResourceRepository, resource);
+ }
+ }
+ try {
+ broker.sendMessageToPrimaryCC(new VoidResponse(reqId, null));
+ } catch (Exception e) {
+ LOGGER.error("failed to notify CC of storage clean up; halting...", e);
+ ExitUtil.halt(EC_NC_FAILED_TO_NOTIFY_TASKS_COMPLETED);
+ }
+ }
+
+ private void deleteInvalidIndex(INcApplicationContext appContext,
+ PersistentLocalResourceRepository localResourceRepository, LocalResource resource)
+ throws HyracksDataException {
+ IDatasetLifecycleManager lcManager = appContext.getDatasetLifecycleManager();
+ String resourceRelPath = resource.getPath();
+ synchronized (lcManager) {
+ IIndex index;
+ index = lcManager.get(resourceRelPath);
+ if (index != null) {
+ LOGGER.warn("unregistering invalid index {}", resourceRelPath);
+ lcManager.unregister(resourceRelPath);
+ } else {
+ LOGGER.warn("initializing unregistered invalid index {}", resourceRelPath);
+ try {
+ index = resource.getResource().createInstance(appContext.getServiceContext());
+ } catch (Exception e) {
+ LOGGER.warn("failed to initialize invalid index {}", resourceRelPath, e);
+ }
+ }
+ localResourceRepository.delete(resourceRelPath);
+ if (index != null) {
+ index.destroy();
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return StorageCleanupRequestMessage.class.getSimpleName();
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/VoidResponse.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/VoidResponse.java
new file mode 100644
index 0000000..6a51c2d
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/VoidResponse.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker.ResponseState;
+import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
+import org.apache.asterix.common.messaging.api.INcResponse;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * A response to a request only indicating success or failure
+ */
+public class VoidResponse implements ICcAddressedMessage, INcResponse {
+
+ private static final long serialVersionUID = 1L;
+ private final Long reqId;
+ private final Throwable failure;
+
+ public VoidResponse(Long reqId, Throwable failure) {
+ this.reqId = reqId;
+ this.failure = failure;
+ }
+
+ @Override
+ public void handle(ICcApplicationContext appCtx) throws HyracksDataException, InterruptedException {
+ ICCMessageBroker broker = (ICCMessageBroker) appCtx.getServiceContext().getMessageBroker();
+ broker.respond(reqId, this);
+ }
+
+ @Override
+ public void setResult(MutablePair<ResponseState, Object> result) {
+ if (failure != null) {
+ result.setLeft(ResponseState.FAILURE);
+ result.setRight(failure);
+ } else {
+ result.setLeft(ResponseState.SUCCESS);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "{ \"response\" : \"" + (failure == null ? "success" : failure.getClass().getSimpleName()) + "\"}";
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index a9a3a3e..9aa433f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -209,15 +209,15 @@
}
localResourceRepository.deleteStorageData();
}
- int maxConcurrentFlushes = storageProperties.getMaxConcurrentFlushes();
- if (maxConcurrentFlushes <= 0) {
- maxConcurrentFlushes = ioManager.getIODevices().size();
+ int maxScheduledFlushes = storageProperties.getMaxScheduledFlushes();
+ if (maxScheduledFlushes <= 0) {
+ maxScheduledFlushes = ioManager.getIODevices().size();
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("The value of maxConcurrentFlushes is not provided. Setting maxConcurrentFlushes = {}.",
- maxConcurrentFlushes);
+ LOGGER.info("The value of maxScheduledFlushes is not provided. Setting maxConcurrentFlushes = {}.",
+ maxScheduledFlushes);
}
}
- virtualBufferCache = new GlobalVirtualBufferCache(allocator, storageProperties, maxConcurrentFlushes);
+ virtualBufferCache = new GlobalVirtualBufferCache(allocator, storageProperties, maxScheduledFlushes);
// Must start vbc now instead of by life cycle component manager (lccm) because lccm happens after
// the metadata bootstrap task
((ILifeCycleComponent) virtualBufferCache).start();
@@ -584,24 +584,24 @@
String schedulerName = storageProperties.getIoScheduler();
int numPartitions = ioManager.getIODevices().size();
- int maxRunningFlushes = storageProperties.getMaxRunningFlushes(numPartitions);
+ int maxConcurrentFlushes = storageProperties.geMaxConcurrentFlushes(numPartitions);
int maxScheduledMerges = storageProperties.getMaxScheduledMerges(numPartitions);
- int maxRunningMerges = storageProperties.getMaxRunningMerges(numPartitions);
+ int maxConcurrentMerges = storageProperties.getMaxConcurrentMerges(numPartitions);
ILSMIOOperationScheduler ioScheduler = null;
if (AsynchronousScheduler.FACTORY.getName().equalsIgnoreCase(schedulerName)) {
ioScheduler = AsynchronousScheduler.FACTORY.createIoScheduler(getServiceContext().getThreadFactory(),
- HaltCallback.INSTANCE, maxRunningFlushes, maxScheduledMerges, maxRunningMerges);
+ HaltCallback.INSTANCE, maxConcurrentFlushes, maxScheduledMerges, maxConcurrentMerges);
} else if (GreedyScheduler.FACTORY.getName().equalsIgnoreCase(schedulerName)) {
ioScheduler = GreedyScheduler.FACTORY.createIoScheduler(getServiceContext().getThreadFactory(),
- HaltCallback.INSTANCE, maxRunningFlushes, maxScheduledMerges, maxRunningMerges);
+ HaltCallback.INSTANCE, maxConcurrentFlushes, maxScheduledMerges, maxConcurrentMerges);
} else {
if (LOGGER.isWarnEnabled()) {
LOGGER.log(Level.WARN,
"Unknown storage I/O scheduler: " + schedulerName + "; defaulting to greedy I/O scheduler.");
}
ioScheduler = GreedyScheduler.FACTORY.createIoScheduler(getServiceContext().getThreadFactory(),
- HaltCallback.INSTANCE, maxRunningFlushes, maxScheduledMerges, maxRunningMerges);
+ HaltCallback.INSTANCE, maxConcurrentFlushes, maxScheduledMerges, maxConcurrentMerges);
}
return ioScheduler;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 1461ef4..65cb36a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -599,22 +599,19 @@
throw new ACIDException(e);
}
long lastLSN = txnContext.getLastLSN();
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("rollbacking transaction log records from " + firstLSN + " to " + lastLSN);
- }
+ boolean infoEnabled = LOGGER.isInfoEnabled();
// check if the transaction actually wrote some logs.
if (firstLSN == TransactionManagementConstants.LogManagerConstants.TERMINAL_LSN || firstLSN > lastLSN) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("no need to roll back as there were no operations by the txn " + txnContext.getTxnId());
+ if (infoEnabled) {
+ LOGGER.info("no need to rollback as there were no operations by " + txnContext.getTxnId());
}
return;
}
-
- // While reading log records from firstLsn to lastLsn, collect uncommitted txn's Lsns
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("collecting loser transaction's LSNs from " + firstLSN + " to " + lastLSN);
+ if (infoEnabled) {
+ LOGGER.info("rolling back transaction log records from " + firstLSN + " to " + lastLSN + " for "
+ + txnContext.getTxnId());
}
-
+ // While reading log records from firstLsn to lastLsn, collect uncommitted txn's Lsns
Map<TxnEntityId, List<Long>> jobLoserEntity2LSNsMap = new HashMap<>();
TxnEntityId tempKeyTxnEntityId = new TxnEntityId(-1, -1, -1, null, -1, false);
int updateLogCount = 0;
@@ -722,7 +719,7 @@
}
}
- if (LOGGER.isInfoEnabled()) {
+ if (infoEnabled) {
LOGGER.info("undone loser transaction's effect");
LOGGER.info("[RecoveryManager's rollback log count] update/entityCommit/undo:" + updateLogCount + "/"
+ entityCommitLogCount + "/" + undoCount);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index ce6e10d..d2e9284 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.translator;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import static org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS;
import java.io.File;
@@ -605,7 +607,7 @@
(ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance();
if (isExternalDataset && mergePolicyFactory.getName().compareTo("correlated-prefix") == 0) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "The correlated-prefix merge policy cannot be used with external dataset.");
+ "The correlated-prefix merge policy cannot be used with external " + dataset() + "s");
}
if (compactionPolicyProperties == null) {
if (mergePolicyFactory.getName().compareTo("no-merge") != 0) {
@@ -842,8 +844,8 @@
TransactionState.COMMIT);
break;
default:
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Unknown dataset type " + dsType);
+ throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE,
+ dataset.getDatasetType().toString());
}
// #. initialize DatasetIdFactory if it is not initialized.
@@ -950,7 +952,8 @@
SourceLocation sourceLoc) throws AlgebricksException {
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- String.format("Dataset %s has to be a record type.", isMetaItemType ? "meta type" : "type"));
+ String.format(StringUtils.capitalize(dataset()) + " %s has to be a record type.",
+ isMetaItemType ? "meta type" : "type"));
}
}
@@ -1448,13 +1451,13 @@
// Check if the dataset is indexible
if (!ExternalIndexingOperations.isIndexible((ExternalDatasetDetails) ds.getDatasetDetails())) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "dataset using " + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
- + " Adapter can't be indexed");
+ dataset() + " using " + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
+ + " adapter can't be indexed");
}
// Check if the name of the index is valid
if (!ExternalIndexingOperations.isValidIndexName(index.getDatasetName(), index.getIndexName())) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "external dataset index name is invalid");
+ "external " + dataset() + " index name is invalid");
}
// Check if the files index exist
@@ -1779,7 +1782,7 @@
if (dataverseName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME)
|| dataverseName.equals(MetadataConstants.METADATA_DATAVERSE_NAME)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- dataverseName + " dataverse can't be dropped");
+ dataverseName + " " + dataverse() + " can't be dropped");
}
lockUtil.dropDataverseBegin(lockManager, metadataProvider.getLocks(), dataverseName);
try {
@@ -2144,7 +2147,7 @@
}
} else if (ExternalIndexingOperations.isFileIndex(index)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Dropping a dataset's files index is not allowed.");
+ "Dropping " + dataset() + " files index is not allowed.");
}
ensureNonPrimaryIndexDrop(index, sourceLoc);
// #. prepare a job to drop the index in NC.
@@ -3045,7 +3048,8 @@
CreateSynonymStatement css = (CreateSynonymStatement) stmt;
DataverseName dataverseName = getActiveDataverseName(css.getDataverseName());
String synonymName = css.getSynonymName();
- DataverseName objectDataverseName = getActiveDataverseName(css.getObjectDataverseName());
+ DataverseName objectDataverseName =
+ css.getObjectDataverseName() != null ? css.getObjectDataverseName() : dataverseName;
String objectName = css.getObjectName();
lockUtil.createSynonymBegin(lockManager, metadataProvider.getLocks(), dataverseName, synonymName);
try {
@@ -3449,8 +3453,9 @@
ActiveEntityEventsListener listener =
(ActiveEntityEventsListener) activeNotificationHandler.getListener(feedId);
if (listener != null && listener.getState() != ActivityState.STOPPED) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Feed " + feedId
- + " is currently active and connected to the following dataset(s) \n" + listener.toString());
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "Feed " + feedId + " is currently active and connected to the following " + dataset() + "(s) \n"
+ + listener.toString());
} else if (listener != null) {
listener.unregister();
}
@@ -3476,7 +3481,7 @@
if (feedPolicy == null) {
if (!stmtFeedPolicyDrop.getIfExists()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Unknown policy " + policyName + " in dataverse " + dataverseName);
+ "Unknown policy " + policyName + " in " + dataverse() + " " + dataverseName);
}
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
return;
@@ -3607,7 +3612,7 @@
feedName, datasetName);
if (fc != null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Feed" + feedName + " is already connected dataset " + datasetName);
+ "Feed" + feedName + " is already connected to " + dataset() + " " + datasetName);
}
fc = new FeedConnection(dataverseName, feedName, datasetName, appliedFunctions, policyName, whereClauseBody,
outputType.getTypeName());
@@ -3691,7 +3696,7 @@
List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
if (indexes.isEmpty()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Cannot compact the extrenal dataset " + datasetName + " because it has no indexes");
+ "Cannot compact the external " + dataset() + " " + datasetName + " because it has no indexes");
}
Dataverse dataverse =
MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dataverseName);
@@ -4004,14 +4009,14 @@
}
// Dataset external ?
if (ds.getDatasetType() != DatasetType.EXTERNAL) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "dataset " + datasetName + " in dataverse " + dataverseName + " is not an external dataset");
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, dataset() + " " + datasetName
+ + " in " + dataverse() + " " + dataverseName + " is not an external " + dataset());
}
// Dataset has indexes ?
indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
if (indexes.isEmpty()) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "External dataset " + datasetName
- + " in dataverse " + dataverseName + " doesn't have any index");
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "External " + dataset() + " "
+ + datasetName + " in " + dataverse() + " " + dataverseName + " doesn't have any index");
}
// Record transaction time
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index 8165316..ae50880 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -18,11 +18,14 @@
*/
package org.apache.asterix.hyracks.bootstrap;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.asterix.app.message.StorageCleanupRequestMessage;
import org.apache.asterix.common.api.IClusterManagementWork;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.cluster.IGlobalRecoveryManager;
@@ -32,6 +35,7 @@
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.messaging.CCMessageBroker;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -51,6 +55,8 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
+
public class GlobalRecoveryManager implements IGlobalRecoveryManager {
private static final Logger LOGGER = LogManager.getLogger();
@@ -111,6 +117,10 @@
LOGGER.info("Starting Global Recovery");
MetadataManager.INSTANCE.init();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ if (appCtx.getStorageProperties().isStorageGlobalCleanup()) {
+ int storageGlobalCleanupTimeout = appCtx.getStorageProperties().getStorageGlobalCleanupTimeout();
+ performGlobalStorageCleanup(mdTxnCtx, storageGlobalCleanupTimeout);
+ }
mdTxnCtx = doRecovery(appCtx, mdTxnCtx);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
recoveryCompleted = true;
@@ -122,6 +132,27 @@
}
}
+ protected void performGlobalStorageCleanup(MetadataTransactionContext mdTxnCtx, int storageGlobalCleanupTimeoutSecs)
+ throws Exception {
+ List<Dataverse> dataverses = MetadataManager.INSTANCE.getDataverses(mdTxnCtx);
+ IntOpenHashSet validDatasetIds = new IntOpenHashSet();
+ for (Dataverse dataverse : dataverses) {
+ List<Dataset> dataverseDatasets =
+ MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverse.getDataverseName());
+ dataverseDatasets.stream().mapToInt(Dataset::getDatasetId).forEach(validDatasetIds::add);
+ }
+ ICcApplicationContext ccAppCtx = (ICcApplicationContext) serviceCtx.getApplicationContext();
+ final List<String> ncs = new ArrayList<>(ccAppCtx.getClusterStateManager().getParticipantNodes());
+ CCMessageBroker messageBroker = (CCMessageBroker) ccAppCtx.getServiceContext().getMessageBroker();
+ long reqId = messageBroker.newRequestId();
+ List<StorageCleanupRequestMessage> requests = new ArrayList<>();
+ for (int i = 0; i < ncs.size(); i++) {
+ requests.add(new StorageCleanupRequestMessage(reqId, validDatasetIds));
+ }
+ messageBroker.sendSyncRequestToNCs(reqId, ncs, requests,
+ TimeUnit.SECONDS.toMillis(storageGlobalCleanupTimeoutSecs));
+ }
+
protected MetadataTransactionContext doRecovery(ICcApplicationContext appCtx, MetadataTransactionContext mdTxnCtx)
throws Exception {
// Loop over datasets
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalStorageCleanupTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalStorageCleanupTest.java
new file mode 100644
index 0000000..84107fb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalStorageCleanupTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import java.io.File;
+
+import org.apache.asterix.app.bootstrap.TestNodeController;
+import org.apache.asterix.test.common.TestHelper;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class GlobalStorageCleanupTest {
+
+ public static final Logger LOGGER = LogManager.getLogger();
+ private static TestNodeController nc;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ System.out.println("SetUp: ");
+ TestHelper.deleteExistingInstanceFiles();
+ String configPath = System.getProperty("user.dir") + File.separator + "src" + File.separator + "test"
+ + File.separator + "resources" + File.separator + "cc.conf";
+ nc = new TestNodeController(configPath, false);
+ }
+
+ @Test
+ public void globalStorageCleanup() throws Exception {
+ nc.init(true);
+ LSMFlushRecoveryTest.nc = nc;
+ LSMFlushRecoveryTest lsmFlushRecoveryTest = new LSMFlushRecoveryTest();
+ lsmFlushRecoveryTest.initializeTestCtx();
+ lsmFlushRecoveryTest.createIndex();
+ lsmFlushRecoveryTest.readIndex();
+ nc.deInit(false);
+ nc.init(false);
+ // the index should deleted after the node initialization
+ lsmFlushRecoveryTest.initializeTestCtx();
+ boolean failedToReadIndex = false;
+ try {
+ lsmFlushRecoveryTest.readIndex();
+ } catch (Exception e) {
+ failedToReadIndex = true;
+ Assert.assertTrue(e.getMessage().contains(ErrorCode.INDEX_DOES_NOT_EXIST.errorCode()));
+ }
+ Assert.assertTrue(failedToReadIndex);
+ nc.deInit(false);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index c3a6839..9c6e95e 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -77,7 +77,7 @@
public class LSMFlushRecoveryTest {
public static final Logger LOGGER = LogManager.getLogger();
- private static TestNodeController nc;
+ public static TestNodeController nc;
private static Dataset dataset;
private static PrimaryIndexInfo[] primaryIndexInfos;
private static SecondaryIndexInfo[] secondaryIndexInfo;
@@ -156,6 +156,10 @@
}
private void initializeNc(boolean cleanUpOnStart) throws Exception {
+ // disable global clean up for this test to allow internal index creation
+ List<Pair<IOption, Object>> opts = new ArrayList<>();
+ opts.add(Pair.of(Option.STORAGE_GLOBAL_CLEANUP, false));
+ nc.setOpts(opts);
nc.init(cleanUpOnStart);
ncAppCtx = nc.getAppRuntimeContext();
// Override the LSMIOScheduler to avoid halting on failure and enable
@@ -177,7 +181,7 @@
dsLifecycleMgr = ncAppCtx.getDatasetLifecycleManager();
}
- private void createIndex() throws Exception {
+ public void createIndex() throws Exception {
dataset = StorageTestUtils.DATASET;
secondaryIndexEntity = new Index(dataset.getDataverseName(), dataset.getDatasetName(), SECONDARY_INDEX_NAME,
SECONDARY_INDEX_TYPE, SECONDARY_INDEX_FIELD_NAMES, SECONDARY_INDEX_FIELD_INDICATORS,
@@ -193,7 +197,7 @@
}
- private void initializeTestCtx() throws Exception {
+ public void initializeTestCtx() throws Exception {
JobId jobId = nc.newJobId();
testCtxs = new IHyracksTaskContext[NUM_PARTITIONS];
for (int i = 0; i < NUM_PARTITIONS; i++) {
@@ -203,7 +207,7 @@
new TransactionOptions(ITransactionManager.AtomicityLevel.ENTITY_LEVEL));
}
- private void readIndex() throws HyracksDataException {
+ public void readIndex() throws HyracksDataException {
primaryIndexDataflowHelpers = new IIndexDataflowHelper[NUM_PARTITIONS];
primaryIndexes = new TestLsmBtree[NUM_PARTITIONS];
for (int i = 0; i < NUM_PARTITIONS; i++) {
diff --git a/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py b/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
index 8b8fced..9058a01 100644
--- a/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
+++ b/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
import math
+import os
def sqrt(num):
return math.sqrt(num)
@@ -26,3 +27,6 @@
def warning(self):
raise ArithmeticError("oof")
+
+ def env_test(self, key):
+ return os.environ[key]
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index e2cd5b9..953284964 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -34,6 +34,7 @@
[nc]
credential.file=src/test/resources/security/passwd
python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
address=127.0.0.1
command=asterixnc
app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp
new file mode 100644
index 0000000..2304349
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type tenkType as closed {
+ unique1 : integer,
+ unique2 : integer,
+ two : integer,
+ four : integer,
+ ten : integer,
+ twenty : integer,
+ hundred : integer,
+ thousand : integer,
+ twothousand : integer,
+ fivethous : integer,
+ tenthous : integer,
+ odd100 : integer,
+ even100 : integer,
+ stringu1 : string,
+ stringu2 : string,
+ string4 : string
+};
+
+create dataset tenk(tenkType) primary key unique2;
+
+SELECT
+ t0.unique1 AS t0_unique1,
+ t1.unique1 AS t1_unique1,
+ t2.unique1 AS t2_unique1
+FROM (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 2
+) t0
+INNER JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 4
+) t1 ON t0.unique2 = t1.unique2
+LEFT JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 6
+) t2 ON t0.unique2 + t2.unique2 = 2 * t1.unique2
+ORDER BY t0_unique1, t1_unique1, t2_unique1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/in_let_8.sqlpp
similarity index 62%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
copy to asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/in_let_8.sqlpp
index 5812379..3699c56 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/in_let_8.sqlpp
@@ -17,8 +17,29 @@
* under the License.
*/
-use tpch;
+drop dataverse test if exists;
+create dataverse test;
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+use test;
+
+SET `compiler.sort.parallel` "false";
+SET `rewrite_internal_queryuid_pk` "false";
+
+create type test.TestType as
+{
+ id : integer
+};
+
+create dataset cart(TestType) primary key id;
+
+select c1.cid, i1.pid, i1.ts
+from cart c1 unnest c1.items i1
+where i1.ts >= 2000 and i1.pid in
+(
+ select value i2.pid
+ from cart c2 unnest c2.items i2
+ where i2.ts >= 2000
+ group by i2.pid
+ having count(*) > 1
+)
+order by c1.cid;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp
index 4494cfa..4299ab7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp
@@ -35,7 +35,7 @@
j AS (
SELECT jid, a
FROM jds
-), ---> 3 rows (jid=1, 2, 3)
+),
m1 AS (
SELECT jid, x, COUNT(1) c1
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
index f6599e6..da74507 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
@@ -50,7 +50,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$309][$$222] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$309] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- UNNEST |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -93,12 +93,13 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- BROADCAST_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
@@ -139,12 +140,13 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- BROADCAST_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan
new file mode 100644
index 0000000..393f1db
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan
@@ -0,0 +1,39 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$136(ASC), $$137(ASC), $#3(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$136(ASC), $$137(ASC), $#3(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$127][$$128] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.tenk.tenk) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.tenk.tenk) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.tenk.tenk) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan
index 29fcd1d..8230bf0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan
@@ -46,20 +46,21 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$146][$$153] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- UNNEST |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$153] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -112,20 +113,21 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$146][$$153] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- UNNEST |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$153] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan
index cb471a9..b81c831 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan
@@ -40,7 +40,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$171] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan
index 52d6e25..2cccba4 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan
@@ -46,7 +46,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$171] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
@@ -123,7 +123,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$171] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
index 3464eae..ec77d62 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
@@ -22,8 +22,8 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$97][$$pid] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan
index a500b92..ee9243d 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan
@@ -22,8 +22,8 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$116][$$108] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$116] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan
index 88cacfa..c670194 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan
@@ -22,8 +22,8 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$116][$$108] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_8.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_8.plan
new file mode 100644
index 0000000..878c508
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_8.plan
@@ -0,0 +1,72 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$101(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$101(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$94, $$i1, $$97, $$93, $$104] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$94(ASC), $$i1(ASC), $$97(ASC), $$93(ASC), $$104(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$97][$$pid] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$94(ASC), $$i1(ASC), $$97(ASC), $$93(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- UNNEST |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.cart) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$112] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$112] |PARTITIONED|
+ -- SORT_GROUP_BY[$$92] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- UNNEST |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.cart) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan
index d631085..5501c67 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan
@@ -40,7 +40,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$175][$$172] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$175] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan
index 5ab7d30..c8944e7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan
@@ -46,7 +46,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$175][$$172] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$175] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
@@ -123,7 +123,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$175][$$172] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$175] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan
index 7cbb67f..4b32064 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan
@@ -7,75 +7,38 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$295][$$296] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$295] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$277][$$jid] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$jid] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$319, $$320] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$319, $$320] |PARTITIONED|
- -- SORT_GROUP_BY[$$273, $$274] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$296] |PARTITIONED|
- -- NESTED_LOOP |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$277, $$240, $$290, $$287, $#4, $$295][$$303, $$308, $$298, $$299, $$300, $$301] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$303][$$306] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
+ -- STABLE_SORT [$$277(ASC), $$240(ASC), $$290(ASC), $$287(ASC), $#4(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$277][$$jid] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$306] |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$jid] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$322, $$323] |PARTITIONED|
+ -- SORT_GROUP_BY[$$324, $$325] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$322, $$323] |PARTITIONED|
- -- SORT_GROUP_BY[$$311, $$312] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$324, $$325] |PARTITIONED|
+ -- SORT_GROUP_BY[$$273, $$274] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
@@ -83,26 +46,74 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$303] |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$303(ASC), $$308(ASC), $$298(ASC), $$299(ASC), $$300(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$303][$$306] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$306] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$327, $$328] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$327, $$328] |PARTITIONED|
+ -- SORT_GROUP_BY[$$311, $$312] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- BROADCAST_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$325, $$326] |PARTITIONED|
+ -- SORT_GROUP_BY[$$330, $$331] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$325, $$326] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$330, $$331] |PARTITIONED|
-- SORT_GROUP_BY[$$275, $$276] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan
index 972e59e..bda21dc 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan
@@ -55,26 +55,26 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$187] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$234] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$234] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$186] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$186(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$233, $$234] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$233, $$234] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$181, $$186] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$181(ASC), $$186(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$170][$$169] |PARTITIONED|
@@ -84,49 +84,50 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$169] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -138,171 +139,173 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$225] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$240] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$240] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$224] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$224(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$239, $$240] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$239, $$240] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$191, $$224] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$191(ASC), $$224(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$173] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$194] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$194] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$194(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$194][$$171] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$195] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$194(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$194][$$171] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$195] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$195][$$197] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$237] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$237] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$203] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$203(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$204][$$207] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$195][$$197] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$197] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$236, $$237] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$236, $$237] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$202, $$203] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$202(ASC), $$203(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$204][$$207] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$173] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
index 1a70178..d61947a 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
@@ -73,26 +73,26 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$221] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$221] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$161] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$161(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$220, $$221] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$220, $$221] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$158, $$161] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$158(ASC), $$161(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$150][$$149] |PARTITIONED|
@@ -102,49 +102,50 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -156,77 +157,162 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$215] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$227] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$227] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$177] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$177(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$226, $$227] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$226, $$227] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$174, $$177] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$174(ASC), $$177(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$152][$$151] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$152] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$178] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$178] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$178(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$178][$$181] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$178(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$178][$$181] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$181] |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$223, $$224] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$223, $$224] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$193, $$194] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$193(ASC), $$194(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$196][$$198] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
@@ -242,89 +328,6 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$224] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$224] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$194] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$194(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$196][$$198] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.1.ddl.sqlpp
deleted file mode 100644
index d3ba2fb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.1.ddl.sqlpp
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-drop dataverse tpch if exists;
-create dataverse tpch;
-
-use tpch;
-
-create type LineItemType as closed {
- l_orderkey: int64,
- l_partkey: int64,
- l_suppkey: int64,
- l_linenumber: int64,
- l_quantity: double,
- l_extendedprice: double,
- l_discount: double,
- l_tax: double,
- l_returnflag: string,
- l_linestatus: string,
- l_shipdate: string,
- l_commitdate: string,
- l_receiptdate: string,
- l_shipinstruct: string,
- l_shipmode: string,
- l_comment: string
-};
-
-create dataset LineItem(LineItemType)
- primary key l_orderkey, l_linenumber;
-
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000..22a30ca
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test1 if exists;
+create external dataset test1(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json"));
+
+drop dataset test2 if exists;
+create external dataset test2(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines/json"),
+("format"="json"));
+
+drop dataset test3 if exists;
+create external dataset test3(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-arrays/json"),
+("format"="json"));
+
+drop dataset test4 if exists;
+create external dataset test4(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-nested-objects/json"),
+("format"="json"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
index 056d96e..7c1708a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
@@ -17,8 +17,7 @@
* under the License.
*/
-use tpch;
+use test;
+set `compiler.externalscanmemory` "8KB";
+select count(*) `count` from test1;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
index 056d96e..d122f42 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
@@ -17,8 +17,7 @@
* under the License.
*/
-use tpch;
+use test;
+set `compiler.externalscanmemory` "16KB";
+select count(*) `count` from test2;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
index 056d96e..552f943 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
@@ -17,8 +17,7 @@
* under the License.
*/
-use tpch;
+use test;
+set `compiler.externalscanmemory` "32KB";
+select count(*) `count` from test3;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
index 056d96e..f30af80 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
@@ -17,8 +17,7 @@
* under the License.
*/
-use tpch;
+use test;
+set `compiler.externalscanmemory` "8KB";
+select value test3 from test3 order by id;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
index 056d96e..af6aff0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
@@ -17,8 +17,7 @@
* under the License.
*/
-use tpch;
+use test;
+set `compiler.externalscanmemory` "16KB";
+select count(*) `count` from test4;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
similarity index 87%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
index 056d96e..854ac80 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
@@ -17,8 +17,7 @@
* under the License.
*/
-use tpch;
+use test;
+set `compiler.externalscanmemory` "10KB";
+select value test4 from test4 order by id;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
index 056d96e..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
@@ -17,8 +17,4 @@
* under the License.
*/
-use tpch;
-
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
index 0ad9fb3..74fe03f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
@@ -23,4 +23,7 @@
as "roundtrip", "Tests.warning" at testlib;
create function roundtrip(s)
- as "roundtrip", "Tests.roundtrip" at testlib;
\ No newline at end of file
+ as "roundtrip", "Tests.roundtrip" at testlib;
+
+create function env_test(k)
+ as "roundtrip", "Tests.env_test" at testlib;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.5.query.sqlpp
similarity index 81%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.5.query.sqlpp
index 5812379..7334470 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.5.query.sqlpp
@@ -16,9 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
+/*
+* Description : Access a records nested records at each level.
+* Expected Res : Success
+* Date : 04 Jun 2015
+*/
+// param max-warnings:json=0
-use tpch;
+use test;
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+env_test("FOO");
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.6.query.sqlpp
similarity index 81%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.6.query.sqlpp
index 5812379..092af06 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.6.query.sqlpp
@@ -16,9 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
+/*
+* Description : Access a records nested records at each level.
+* Expected Res : Success
+* Date : 04 Jun 2015
+*/
+// param max-warnings:json=0
-use tpch;
+use test;
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+env_test("BAR");
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp
new file mode 100644
index 0000000..87b5d75
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type tenkType as closed {
+ unique1 : integer,
+ unique2 : integer,
+ two : integer,
+ four : integer,
+ ten : integer,
+ twenty : integer,
+ hundred : integer,
+ thousand : integer,
+ twothousand : integer,
+ fivethous : integer,
+ tenthous : integer,
+ odd100 : integer,
+ even100 : integer,
+ stringu1 : string,
+ stringu2 : string,
+ string4 : string
+};
+
+create dataset tenk(tenkType) primary key unique2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp
similarity index 85%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp
index 056d96e..2d7e768 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp
@@ -17,8 +17,6 @@
* under the License.
*/
-use tpch;
+use test;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
+load dataset tenk using localfs ((`path`=`asterix_nc1://data/tenk.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp
similarity index 61%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp
index 5812379..823a540 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp
@@ -17,8 +17,23 @@
* under the License.
*/
-use tpch;
+/*
+ * Test nested loop implementation of left outer join
+ */
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+USE test;
+
+SELECT
+ t0.unique1 AS t0_unique1,
+ t1.unique1 AS t1_unique1,
+ t2.unique1 AS t2_unique1
+FROM (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 2
+) t0
+INNER JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 4
+) t1 ON t0.unique2 = t1.unique2
+LEFT JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 6
+) t2 ON t0.unique2 + t2.unique2 = 2 * t1.unique2
+ORDER BY t0_unique1, t1_unique1, t2_unique1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/in_let/in_let.8.query.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/in_let/in_let.8.query.sqlpp
index 5812379..77f6027 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/in_let/in_let.8.query.sqlpp
@@ -17,8 +17,23 @@
* under the License.
*/
-use tpch;
+/*
+ * Test "rewrite_internal_queryuid_pk" option (default is true)
+ */
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+use test;
+
+SET `compiler.sort.parallel` "false";
+SET `rewrite_internal_queryuid_pk` "false";
+
+select c1.cid, i1.pid, i1.ts
+from cart c1 unnest c1.items i1
+where i1.ts >= 2000 and i1.pid in
+(
+ select value i2.pid
+ from cart c2 unnest c2.items i2
+ where i2.ts >= 2000
+ group by i2.pid
+ having count(*) > 1
+)
+order by c1.cid;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.1.ddl.sqlpp
similarity index 69%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.1.ddl.sqlpp
index 5812379..33397c9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.1.ddl.sqlpp
@@ -17,8 +17,19 @@
* under the License.
*/
-use tpch;
+/*
+ * Test that if the target object's dataverse is not specified then
+ * it defaults to the synonym's dataverse
+ */
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+drop dataverse test1 if exists;
+create dataverse test1;
+
+drop dataverse test2 if exists;
+create dataverse test2;
+
+use test1;
+
+create synonym test2.md_syn_1 for `Metadata`.`Synonym`;
+
+create synonym test2.md_syn_2 for md_syn_1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.2.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.2.query.sqlpp
index 056d96e..8e0aaf9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.2.query.sqlpp
@@ -17,8 +17,7 @@
* under the License.
*/
-use tpch;
-
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
+select DataverseName, SynonymName, ObjectDataverseName, ObjectName
+from test2.md_syn_2
+where DataverseName like "test%"
+order by SynonymName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 0df55e5..473319c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -25,6 +25,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "INFO",
"max\.wait\.active\.cluster" : 60,
@@ -44,6 +45,7 @@
"replication\.timeout" : 30,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
+ "storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index e00f4aa..171ead0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -25,6 +25,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
@@ -44,6 +45,7 @@
"replication\.timeout" : 30,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
+ "storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index c847562..4a6aff5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -25,6 +25,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
@@ -44,6 +45,7 @@
"replication\.timeout" : 30,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
+ "storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/binary/query_id/query_id.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/binary/query_id/query_id.1.adm
deleted file mode 100644
index dcfde8d..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/binary/query_id/query_id.1.adm
+++ /dev/null
@@ -1,6005 +0,0 @@
-"000000000000000000000001"
-"000000000000000000000002"
-"000000000000000000000003"
-"000000000000000000000004"
-"000000000000000000000005"
-"000000000000000000000006"
-"000000000000000000000007"
-"000000000000000000000008"
-"000000000000000000000009"
-"00000000000000000000000A"
-"00000000000000000000000B"
-"00000000000000000000000C"
-"00000000000000000000000D"
-"00000000000000000000000E"
-"00000000000000000000000F"
-"000000000000000000000010"
-"000000000000000000000011"
-"000000000000000000000012"
-"000000000000000000000013"
-"000000000000000000000014"
-"000000000000000000000015"
-"000000000000000000000016"
-"000000000000000000000017"
-"000000000000000000000018"
-"000000000000000000000019"
-"00000000000000000000001A"
-"00000000000000000000001B"
-"00000000000000000000001C"
-"00000000000000000000001D"
-"00000000000000000000001E"
-"00000000000000000000001F"
-"000000000000000000000020"
-"000000000000000000000021"
-"000000000000000000000022"
-"000000000000000000000023"
-"000000000000000000000024"
-"000000000000000000000025"
-"000000000000000000000026"
-"000000000000000000000027"
-"000000000000000000000028"
-"000000000000000000000029"
-"00000000000000000000002A"
-"00000000000000000000002B"
-"00000000000000000000002C"
-"00000000000000000000002D"
-"00000000000000000000002E"
-"00000000000000000000002F"
-"000000000000000000000030"
-"000000000000000000000031"
-"000000000000000000000032"
-"000000000000000000000033"
-"000000000000000000000034"
-"000000000000000000000035"
-"000000000000000000000036"
-"000000000000000000000037"
-"000000000000000000000038"
-"000000000000000000000039"
-"00000000000000000000003A"
-"00000000000000000000003B"
-"00000000000000000000003C"
-"00000000000000000000003D"
-"00000000000000000000003E"
-"00000000000000000000003F"
-"000000000000000000000040"
-"000000000000000000000041"
-"000000000000000000000042"
-"000000000000000000000043"
-"000000000000000000000044"
-"000000000000000000000045"
-"000000000000000000000046"
-"000000000000000000000047"
-"000000000000000000000048"
-"000000000000000000000049"
-"00000000000000000000004A"
-"00000000000000000000004B"
-"00000000000000000000004C"
-"00000000000000000000004D"
-"00000000000000000000004E"
-"00000000000000000000004F"
-"000000000000000000000050"
-"000000000000000000000051"
-"000000000000000000000052"
-"000000000000000000000053"
-"000000000000000000000054"
-"000000000000000000000055"
-"000000000000000000000056"
-"000000000000000000000057"
-"000000000000000000000058"
-"000000000000000000000059"
-"00000000000000000000005A"
-"00000000000000000000005B"
-"00000000000000000000005C"
-"00000000000000000000005D"
-"00000000000000000000005E"
-"00000000000000000000005F"
-"000000000000000000000060"
-"000000000000000000000061"
-"000000000000000000000062"
-"000000000000000000000063"
-"000000000000000000000064"
-"000000000000000000000065"
-"000000000000000000000066"
-"000000000000000000000067"
-"000000000000000000000068"
-"000000000000000000000069"
-"00000000000000000000006A"
-"00000000000000000000006B"
-"00000000000000000000006C"
-"00000000000000000000006D"
-"00000000000000000000006E"
-"00000000000000000000006F"
-"000000000000000000000070"
-"000000000000000000000071"
-"000000000000000000000072"
-"000000000000000000000073"
-"000000000000000000000074"
-"000000000000000000000075"
-"000000000000000000000076"
-"000000000000000000000077"
-"000000000000000000000078"
-"000000000000000000000079"
-"00000000000000000000007A"
-"00000000000000000000007B"
-"00000000000000000000007C"
-"00000000000000000000007D"
-"00000000000000000000007E"
-"00000000000000000000007F"
-"000000000000000000000080"
-"000000000000000000000081"
-"000000000000000000000082"
-"000000000000000000000083"
-"000000000000000000000084"
-"000000000000000000000085"
-"000000000000000000000086"
-"000000000000000000000087"
-"000000000000000000000088"
-"000000000000000000000089"
-"00000000000000000000008A"
-"00000000000000000000008B"
-"00000000000000000000008C"
-"00000000000000000000008D"
-"00000000000000000000008E"
-"00000000000000000000008F"
-"000000000000000000000090"
-"000000000000000000000091"
-"000000000000000000000092"
-"000000000000000000000093"
-"000000000000000000000094"
-"000000000000000000000095"
-"000000000000000000000096"
-"000000000000000000000097"
-"000000000000000000000098"
-"000000000000000000000099"
-"00000000000000000000009A"
-"00000000000000000000009B"
-"00000000000000000000009C"
-"00000000000000000000009D"
-"00000000000000000000009E"
-"00000000000000000000009F"
-"0000000000000000000000A0"
-"0000000000000000000000A1"
-"0000000000000000000000A2"
-"0000000000000000000000A3"
-"0000000000000000000000A4"
-"0000000000000000000000A5"
-"0000000000000000000000A6"
-"0000000000000000000000A7"
-"0000000000000000000000A8"
-"0000000000000000000000A9"
-"0000000000000000000000AA"
-"0000000000000000000000AB"
-"0000000000000000000000AC"
-"0000000000000000000000AD"
-"0000000000000000000000AE"
-"0000000000000000000000AF"
-"0000000000000000000000B0"
-"0000000000000000000000B1"
-"0000000000000000000000B2"
-"0000000000000000000000B3"
-"0000000000000000000000B4"
-"0000000000000000000000B5"
-"0000000000000000000000B6"
-"0000000000000000000000B7"
-"0000000000000000000000B8"
-"0000000000000000000000B9"
-"0000000000000000000000BA"
-"0000000000000000000000BB"
-"0000000000000000000000BC"
-"0000000000000000000000BD"
-"0000000000000000000000BE"
-"0000000000000000000000BF"
-"0000000000000000000000C0"
-"0000000000000000000000C1"
-"0000000000000000000000C2"
-"0000000000000000000000C3"
-"0000000000000000000000C4"
-"0000000000000000000000C5"
-"0000000000000000000000C6"
-"0000000000000000000000C7"
-"0000000000000000000000C8"
-"0000000000000000000000C9"
-"0000000000000000000000CA"
-"0000000000000000000000CB"
-"0000000000000000000000CC"
-"0000000000000000000000CD"
-"0000000000000000000000CE"
-"0000000000000000000000CF"
-"0000000000000000000000D0"
-"0000000000000000000000D1"
-"0000000000000000000000D2"
-"0000000000000000000000D3"
-"0000000000000000000000D4"
-"0000000000000000000000D5"
-"0000000000000000000000D6"
-"0000000000000000000000D7"
-"0000000000000000000000D8"
-"0000000000000000000000D9"
-"0000000000000000000000DA"
-"0000000000000000000000DB"
-"0000000000000000000000DC"
-"0000000000000000000000DD"
-"0000000000000000000000DE"
-"0000000000000000000000DF"
-"0000000000000000000000E0"
-"0000000000000000000000E1"
-"0000000000000000000000E2"
-"0000000000000000000000E3"
-"0000000000000000000000E4"
-"0000000000000000000000E5"
-"0000000000000000000000E6"
-"0000000000000000000000E7"
-"0000000000000000000000E8"
-"0000000000000000000000E9"
-"0000000000000000000000EA"
-"0000000000000000000000EB"
-"0000000000000000000000EC"
-"0000000000000000000000ED"
-"0000000000000000000000EE"
-"0000000000000000000000EF"
-"0000000000000000000000F0"
-"0000000000000000000000F1"
-"0000000000000000000000F2"
-"0000000000000000000000F3"
-"0000000000000000000000F4"
-"0000000000000000000000F5"
-"0000000000000000000000F6"
-"0000000000000000000000F7"
-"0000000000000000000000F8"
-"0000000000000000000000F9"
-"0000000000000000000000FA"
-"0000000000000000000000FB"
-"0000000000000000000000FC"
-"0000000000000000000000FD"
-"0000000000000000000000FE"
-"0000000000000000000000FF"
-"000000000000000000000100"
-"000000000000000000000101"
-"000000000000000000000102"
-"000000000000000000000103"
-"000000000000000000000104"
-"000000000000000000000105"
-"000000000000000000000106"
-"000000000000000000000107"
-"000000000000000000000108"
-"000000000000000000000109"
-"00000000000000000000010A"
-"00000000000000000000010B"
-"00000000000000000000010C"
-"00000000000000000000010D"
-"00000000000000000000010E"
-"00000000000000000000010F"
-"000000000000000000000110"
-"000000000000000000000111"
-"000000000000000000000112"
-"000000000000000000000113"
-"000000000000000000000114"
-"000000000000000000000115"
-"000000000000000000000116"
-"000000000000000000000117"
-"000000000000000000000118"
-"000000000000000000000119"
-"00000000000000000000011A"
-"00000000000000000000011B"
-"00000000000000000000011C"
-"00000000000000000000011D"
-"00000000000000000000011E"
-"00000000000000000000011F"
-"000000000000000000000120"
-"000000000000000000000121"
-"000000000000000000000122"
-"000000000000000000000123"
-"000000000000000000000124"
-"000000000000000000000125"
-"000000000000000000000126"
-"000000000000000000000127"
-"000000000000000000000128"
-"000000000000000000000129"
-"00000000000000000000012A"
-"00000000000000000000012B"
-"00000000000000000000012C"
-"00000000000000000000012D"
-"00000000000000000000012E"
-"00000000000000000000012F"
-"000000000000000000000130"
-"000000000000000000000131"
-"000000000000000000000132"
-"000000000000000000000133"
-"000000000000000000000134"
-"000000000000000000000135"
-"000000000000000000000136"
-"000000000000000000000137"
-"000000000000000000000138"
-"000000000000000000000139"
-"00000000000000000000013A"
-"00000000000000000000013B"
-"00000000000000000000013C"
-"00000000000000000000013D"
-"00000000000000000000013E"
-"00000000000000000000013F"
-"000000000000000000000140"
-"000000000000000000000141"
-"000000000000000000000142"
-"000000000000000000000143"
-"000000000000000000000144"
-"000000000000000000000145"
-"000000000000000000000146"
-"000000000000000000000147"
-"000000000000000000000148"
-"000000000000000000000149"
-"00000000000000000000014A"
-"00000000000000000000014B"
-"00000000000000000000014C"
-"00000000000000000000014D"
-"00000000000000000000014E"
-"00000000000000000000014F"
-"000000000000000000000150"
-"000000000000000000000151"
-"000000000000000000000152"
-"000000000000000000000153"
-"000000000000000000000154"
-"000000000000000000000155"
-"000000000000000000000156"
-"000000000000000000000157"
-"000000000000000000000158"
-"000000000000000000000159"
-"00000000000000000000015A"
-"00000000000000000000015B"
-"00000000000000000000015C"
-"00000000000000000000015D"
-"00000000000000000000015E"
-"00000000000000000000015F"
-"000000000000000000000160"
-"000000000000000000000161"
-"000000000000000000000162"
-"000000000000000000000163"
-"000000000000000000000164"
-"000000000000000000000165"
-"000000000000000000000166"
-"000000000000000000000167"
-"000000000000000000000168"
-"000000000000000000000169"
-"00000000000000000000016A"
-"00000000000000000000016B"
-"00000000000000000000016C"
-"00000000000000000000016D"
-"00000000000000000000016E"
-"00000000000000000000016F"
-"000000000000000000000170"
-"000000000000000000000171"
-"000000000000000000000172"
-"000000000000000000000173"
-"000000000000000000000174"
-"000000000000000000000175"
-"000000000000000000000176"
-"000000000000000000000177"
-"000000000000000000000178"
-"000000000000000000000179"
-"00000000000000000000017A"
-"00000000000000000000017B"
-"00000000000000000000017C"
-"00000000000000000000017D"
-"00000000000000000000017E"
-"00000000000000000000017F"
-"000000000000000000000180"
-"000000000000000000000181"
-"000000000000000000000182"
-"000000000000000000000183"
-"000000000000000000000184"
-"000000000000000000000185"
-"000000000000000000000186"
-"000000000000000000000187"
-"000000000000000000000188"
-"000000000000000000000189"
-"00000000000000000000018A"
-"00000000000000000000018B"
-"00000000000000000000018C"
-"00000000000000000000018D"
-"00000000000000000000018E"
-"00000000000000000000018F"
-"000000000000000000000190"
-"000000000000000000000191"
-"000000000000000000000192"
-"000000000000000000000193"
-"000000000000000000000194"
-"000000000000000000000195"
-"000000000000000000000196"
-"000000000000000000000197"
-"000000000000000000000198"
-"000000000000000000000199"
-"00000000000000000000019A"
-"00000000000000000000019B"
-"00000000000000000000019C"
-"00000000000000000000019D"
-"00000000000000000000019E"
-"00000000000000000000019F"
-"0000000000000000000001A0"
-"0000000000000000000001A1"
-"0000000000000000000001A2"
-"0000000000000000000001A3"
-"0000000000000000000001A4"
-"0000000000000000000001A5"
-"0000000000000000000001A6"
-"0000000000000000000001A7"
-"0000000000000000000001A8"
-"0000000000000000000001A9"
-"0000000000000000000001AA"
-"0000000000000000000001AB"
-"0000000000000000000001AC"
-"0000000000000000000001AD"
-"0000000000000000000001AE"
-"0000000000000000000001AF"
-"0000000000000000000001B0"
-"0000000000000000000001B1"
-"0000000000000000000001B2"
-"0000000000000000000001B3"
-"0000000000000000000001B4"
-"0000000000000000000001B5"
-"0000000000000000000001B6"
-"0000000000000000000001B7"
-"0000000000000000000001B8"
-"0000000000000000000001B9"
-"0000000000000000000001BA"
-"0000000000000000000001BB"
-"0000000000000000000001BC"
-"0000000000000000000001BD"
-"0000000000000000000001BE"
-"0000000000000000000001BF"
-"0000000000000000000001C0"
-"0000000000000000000001C1"
-"0000000000000000000001C2"
-"0000000000000000000001C3"
-"0000000000000000000001C4"
-"0000000000000000000001C5"
-"0000000000000000000001C6"
-"0000000000000000000001C7"
-"0000000000000000000001C8"
-"0000000000000000000001C9"
-"0000000000000000000001CA"
-"0000000000000000000001CB"
-"0000000000000000000001CC"
-"0000000000000000000001CD"
-"0000000000000000000001CE"
-"0000000000000000000001CF"
-"0000000000000000000001D0"
-"0000000000000000000001D1"
-"0000000000000000000001D2"
-"0000000000000000000001D3"
-"0000000000000000000001D4"
-"0000000000000000000001D5"
-"0000000000000000000001D6"
-"0000000000000000000001D7"
-"0000000000000000000001D8"
-"0000000000000000000001D9"
-"0000000000000000000001DA"
-"0000000000000000000001DB"
-"0000000000000000000001DC"
-"0000000000000000000001DD"
-"0000000000000000000001DE"
-"0000000000000000000001DF"
-"0000000000000000000001E0"
-"0000000000000000000001E1"
-"0000000000000000000001E2"
-"0000000000000000000001E3"
-"0000000000000000000001E4"
-"0000000000000000000001E5"
-"0000000000000000000001E6"
-"0000000000000000000001E7"
-"0000000000000000000001E8"
-"0000000000000000000001E9"
-"0000000000000000000001EA"
-"0000000000000000000001EB"
-"0000000000000000000001EC"
-"0000000000000000000001ED"
-"0000000000000000000001EE"
-"0000000000000000000001EF"
-"0000000000000000000001F0"
-"0000000000000000000001F1"
-"0000000000000000000001F2"
-"0000000000000000000001F3"
-"0000000000000000000001F4"
-"0000000000000000000001F5"
-"0000000000000000000001F6"
-"0000000000000000000001F7"
-"0000000000000000000001F8"
-"0000000000000000000001F9"
-"0000000000000000000001FA"
-"0000000000000000000001FB"
-"0000000000000000000001FC"
-"0000000000000000000001FD"
-"0000000000000000000001FE"
-"0000000000000000000001FF"
-"000000000000000000000200"
-"000000000000000000000201"
-"000000000000000000000202"
-"000000000000000000000203"
-"000000000000000000000204"
-"000000000000000000000205"
-"000000000000000000000206"
-"000000000000000000000207"
-"000000000000000000000208"
-"000000000000000000000209"
-"00000000000000000000020A"
-"00000000000000000000020B"
-"00000000000000000000020C"
-"00000000000000000000020D"
-"00000000000000000000020E"
-"00000000000000000000020F"
-"000000000000000000000210"
-"000000000000000000000211"
-"000000000000000000000212"
-"000000000000000000000213"
-"000000000000000000000214"
-"000000000000000000000215"
-"000000000000000000000216"
-"000000000000000000000217"
-"000000000000000000000218"
-"000000000000000000000219"
-"00000000000000000000021A"
-"00000000000000000000021B"
-"00000000000000000000021C"
-"00000000000000000000021D"
-"00000000000000000000021E"
-"00000000000000000000021F"
-"000000000000000000000220"
-"000000000000000000000221"
-"000000000000000000000222"
-"000000000000000000000223"
-"000000000000000000000224"
-"000000000000000000000225"
-"000000000000000000000226"
-"000000000000000000000227"
-"000000000000000000000228"
-"000000000000000000000229"
-"00000000000000000000022A"
-"00000000000000000000022B"
-"00000000000000000000022C"
-"00000000000000000000022D"
-"00000000000000000000022E"
-"00000000000000000000022F"
-"000000000000000000000230"
-"000000000000000000000231"
-"000000000000000000000232"
-"000000000000000000000233"
-"000000000000000000000234"
-"000000000000000000000235"
-"000000000000000000000236"
-"000000000000000000000237"
-"000000000000000000000238"
-"000000000000000000000239"
-"00000000000000000000023A"
-"00000000000000000000023B"
-"00000000000000000000023C"
-"00000000000000000000023D"
-"00000000000000000000023E"
-"00000000000000000000023F"
-"000000000000000000000240"
-"000000000000000000000241"
-"000000000000000000000242"
-"000000000000000000000243"
-"000000000000000000000244"
-"000000000000000000000245"
-"000000000000000000000246"
-"000000000000000000000247"
-"000000000000000000000248"
-"000000000000000000000249"
-"00000000000000000000024A"
-"00000000000000000000024B"
-"00000000000000000000024C"
-"00000000000000000000024D"
-"00000000000000000000024E"
-"00000000000000000000024F"
-"000000000000000000000250"
-"000000000000000000000251"
-"000000000000000000000252"
-"000000000000000000000253"
-"000000000000000000000254"
-"000000000000000000000255"
-"000000000000000000000256"
-"000000000000000000000257"
-"000000000000000000000258"
-"000000000000000000000259"
-"00000000000000000000025A"
-"00000000000000000000025B"
-"00000000000000000000025C"
-"00000000000000000000025D"
-"00000000000000000000025E"
-"00000000000000000000025F"
-"000000000000000000000260"
-"000000000000000000000261"
-"000000000000000000000262"
-"000000000000000000000263"
-"000000000000000000000264"
-"000000000000000000000265"
-"000000000000000000000266"
-"000000000000000000000267"
-"000000000000000000000268"
-"000000000000000000000269"
-"00000000000000000000026A"
-"00000000000000000000026B"
-"00000000000000000000026C"
-"00000000000000000000026D"
-"00000000000000000000026E"
-"00000000000000000000026F"
-"000000000000000000000270"
-"000000000000000000000271"
-"000000000000000000000272"
-"000000000000000000000273"
-"000000000000000000000274"
-"000000000000000000000275"
-"000000000000000000000276"
-"000000000000000000000277"
-"000000000000000000000278"
-"000000000000000000000279"
-"00000000000000000000027A"
-"00000000000000000000027B"
-"00000000000000000000027C"
-"00000000000000000000027D"
-"00000000000000000000027E"
-"00000000000000000000027F"
-"000000000000000000000280"
-"000000000000000000000281"
-"000000000000000000000282"
-"000000000000000000000283"
-"000000000000000000000284"
-"000000000000000000000285"
-"000000000000000000000286"
-"000000000000000000000287"
-"000000000000000000000288"
-"000000000000000000000289"
-"00000000000000000000028A"
-"00000000000000000000028B"
-"00000000000000000000028C"
-"00000000000000000000028D"
-"00000000000000000000028E"
-"00000000000000000000028F"
-"000000000000000000000290"
-"000000000000000000000291"
-"000000000000000000000292"
-"000000000000000000000293"
-"000000000000000000000294"
-"000000000000000000000295"
-"000000000000000000000296"
-"000000000000000000000297"
-"000000000000000000000298"
-"000000000000000000000299"
-"00000000000000000000029A"
-"00000000000000000000029B"
-"00000000000000000000029C"
-"00000000000000000000029D"
-"00000000000000000000029E"
-"00000000000000000000029F"
-"0000000000000000000002A0"
-"0000000000000000000002A1"
-"0000000000000000000002A2"
-"0000000000000000000002A3"
-"0000000000000000000002A4"
-"0000000000000000000002A5"
-"0000000000000000000002A6"
-"0000000000000000000002A7"
-"0000000000000000000002A8"
-"0000000000000000000002A9"
-"0000000000000000000002AA"
-"0000000000000000000002AB"
-"0000000000000000000002AC"
-"0000000000000000000002AD"
-"0000000000000000000002AE"
-"0000000000000000000002AF"
-"0000000000000000000002B0"
-"0000000000000000000002B1"
-"0000000000000000000002B2"
-"0000000000000000000002B3"
-"0000000000000000000002B4"
-"0000000000000000000002B5"
-"0000000000000000000002B6"
-"0000000000000000000002B7"
-"0000000000000000000002B8"
-"0000000000000000000002B9"
-"0000000000000000000002BA"
-"0000000000000000000002BB"
-"0000000000000000000002BC"
-"0000000000000000000002BD"
-"0000000000000000000002BE"
-"0000000000000000000002BF"
-"0000000000000000000002C0"
-"0000000000000000000002C1"
-"0000000000000000000002C2"
-"0000000000000000000002C3"
-"0000000000000000000002C4"
-"0000000000000000000002C5"
-"0000000000000000000002C6"
-"0000000000000000000002C7"
-"0000000000000000000002C8"
-"0000000000000000000002C9"
-"0000000000000000000002CA"
-"0000000000000000000002CB"
-"0000000000000000000002CC"
-"0000000000000000000002CD"
-"0000000000000000000002CE"
-"0000000000000000000002CF"
-"0000000000000000000002D0"
-"0000000000000000000002D1"
-"0000000000000000000002D2"
-"0000000000000000000002D3"
-"0000000000000000000002D4"
-"0000000000000000000002D5"
-"0000000000000000000002D6"
-"0000000000000000000002D7"
-"0000000000000000000002D8"
-"0000000000000000000002D9"
-"0000000000000000000002DA"
-"0000000000000000000002DB"
-"0000000000000000000002DC"
-"0000000000000000000002DD"
-"0000000000000000000002DE"
-"0000000000000000000002DF"
-"0000000000000000000002E0"
-"0000000000000000000002E1"
-"0000000000000000000002E2"
-"0000000000000000000002E3"
-"0000000000000000000002E4"
-"0000000000000000000002E5"
-"0000000000000000000002E6"
-"0000000000000000000002E7"
-"0000000000000000000002E8"
-"0000000000000000000002E9"
-"0000000000000000000002EA"
-"0000000000000000000002EB"
-"0000000000000000000002EC"
-"0000000000000000000002ED"
-"0000000000000000000002EE"
-"0000000000000000000002EF"
-"0000000000000000000002F0"
-"0000000000000000000002F1"
-"0000000000000000000002F2"
-"0000000000000000000002F3"
-"0000000000000000000002F4"
-"0000000000000000000002F5"
-"0000000000000000000002F6"
-"0000000000000000000002F7"
-"0000000000000000000002F8"
-"0000000000000000000002F9"
-"0000000000000000000002FA"
-"0000000000000000000002FB"
-"0000000000000000000002FC"
-"0000000000000000000002FD"
-"0000000000000000000002FE"
-"0000000000000000000002FF"
-"000000000000000000000300"
-"000000000000000000000301"
-"000000000000000000000302"
-"000000000000000000000303"
-"000000000000000000000304"
-"000000000000000000000305"
-"000000000000000000000306"
-"000000000000000000000307"
-"000000000000000000000308"
-"000000000000000000000309"
-"00000000000000000000030A"
-"00000000000000000000030B"
-"00000000000000000000030C"
-"00000000000000000000030D"
-"00000000000000000000030E"
-"00000000000000000000030F"
-"000000000000000000000310"
-"000000000000000000000311"
-"000000000000000000000312"
-"000000000000000000000313"
-"000000000000000000000314"
-"000000000000000000000315"
-"000000000000000000000316"
-"000000000000000000000317"
-"000000000000000000000318"
-"000000000000000000000319"
-"00000000000000000000031A"
-"00000000000000000000031B"
-"00000000000000000000031C"
-"00000000000000000000031D"
-"00000000000000000000031E"
-"00000000000000000000031F"
-"000000000000000000000320"
-"000000000000000000000321"
-"000000000000000000000322"
-"000000000000000000000323"
-"000000000000000000000324"
-"000000000000000000000325"
-"000000000000000000000326"
-"000000000000000000000327"
-"000000000000000000000328"
-"000000000000000000000329"
-"00000000000000000000032A"
-"00000000000000000000032B"
-"00000000000000000000032C"
-"00000000000000000000032D"
-"00000000000000000000032E"
-"00000000000000000000032F"
-"000000000000000000000330"
-"000000000000000000000331"
-"000000000000000000000332"
-"000000000000000000000333"
-"000000000000000000000334"
-"000000000000000000000335"
-"000000000000000000000336"
-"000000000000000000000337"
-"000000000000000000000338"
-"000000000000000000000339"
-"00000000000000000000033A"
-"00000000000000000000033B"
-"00000000000000000000033C"
-"00000000000000000000033D"
-"00000000000000000000033E"
-"00000000000000000000033F"
-"000000000000000000000340"
-"000000000000000000000341"
-"000000000000000000000342"
-"000000000000000000000343"
-"000000000000000000000344"
-"000000000000000000000345"
-"000000000000000000000346"
-"000000000000000000000347"
-"000000000000000000000348"
-"000000000000000000000349"
-"00000000000000000000034A"
-"00000000000000000000034B"
-"00000000000000000000034C"
-"00000000000000000000034D"
-"00000000000000000000034E"
-"00000000000000000000034F"
-"000000000000000000000350"
-"000000000000000000000351"
-"000000000000000000000352"
-"000000000000000000000353"
-"000000000000000000000354"
-"000000000000000000000355"
-"000000000000000000000356"
-"000000000000000000000357"
-"000000000000000000000358"
-"000000000000000000000359"
-"00000000000000000000035A"
-"00000000000000000000035B"
-"00000000000000000000035C"
-"00000000000000000000035D"
-"00000000000000000000035E"
-"00000000000000000000035F"
-"000000000000000000000360"
-"000000000000000000000361"
-"000000000000000000000362"
-"000000000000000000000363"
-"000000000000000000000364"
-"000000000000000000000365"
-"000000000000000000000366"
-"000000000000000000000367"
-"000000000000000000000368"
-"000000000000000000000369"
-"00000000000000000000036A"
-"00000000000000000000036B"
-"00000000000000000000036C"
-"00000000000000000000036D"
-"00000000000000000000036E"
-"00000000000000000000036F"
-"000000000000000000000370"
-"000000000000000000000371"
-"000000000000000000000372"
-"000000000000000000000373"
-"000000000000000000000374"
-"000000000000000000000375"
-"000000000000000000000376"
-"000000000000000000000377"
-"000000000000000000000378"
-"000000000000000000000379"
-"00000000000000000000037A"
-"00000000000000000000037B"
-"00000000000000000000037C"
-"00000000000000000000037D"
-"00000000000000000000037E"
-"00000000000000000000037F"
-"000000000000000000000380"
-"000000000000000000000381"
-"000000000000000000000382"
-"000000000000000000000383"
-"000000000000000000000384"
-"000000000000000000000385"
-"000000000000000000000386"
-"000000000000000000000387"
-"000000000000000000000388"
-"000000000000000000000389"
-"00000000000000000000038A"
-"00000000000000000000038B"
-"00000000000000000000038C"
-"00000000000000000000038D"
-"00000000000000000000038E"
-"00000000000000000000038F"
-"000000000000000000000390"
-"000000000000000000000391"
-"000000000000000000000392"
-"000000000000000000000393"
-"000000000000000000000394"
-"000000000000000000000395"
-"000000000000000000000396"
-"000000000000000000000397"
-"000000000000000000000398"
-"000000000000000000000399"
-"00000000000000000000039A"
-"00000000000000000000039B"
-"00000000000000000000039C"
-"00000000000000000000039D"
-"00000000000000000000039E"
-"00000000000000000000039F"
-"0000000000000000000003A0"
-"0000000000000000000003A1"
-"0000000000000000000003A2"
-"0000000000000000000003A3"
-"0000000000000000000003A4"
-"0000000000000000000003A5"
-"0000000000000000000003A6"
-"0000000000000000000003A7"
-"0000000000000000000003A8"
-"0000000000000000000003A9"
-"0000000000000000000003AA"
-"0000000000000000000003AB"
-"0000000000000000000003AC"
-"0000000000000000000003AD"
-"0000000000000000000003AE"
-"0000000000000000000003AF"
-"0000000000000000000003B0"
-"0000000000000000000003B1"
-"0000000000000000000003B2"
-"0000000000000000000003B3"
-"0000000000000000000003B4"
-"0000000000000000000003B5"
-"0000000000000000000003B6"
-"0000000000000000000003B7"
-"0000000000000000000003B8"
-"0000000000000000000003B9"
-"0000000000000000000003BA"
-"0000000000000000000003BB"
-"0000000000000000000003BC"
-"0000000000000000000003BD"
-"0000000000000000000003BE"
-"0000000000000000000003BF"
-"0000000000000000000003C0"
-"0000000000000000000003C1"
-"0000000000000000000003C2"
-"0000000000000000000003C3"
-"0000000000000000000003C4"
-"0000000000000000000003C5"
-"0000000000000000000003C6"
-"0000000000000000000003C7"
-"0000000000000000000003C8"
-"0000000000000000000003C9"
-"0000000000000000000003CA"
-"0000000000000000000003CB"
-"0000000000000000000003CC"
-"0000000000000000000003CD"
-"0000000000000000000003CE"
-"0000000000000000000003CF"
-"0000000000000000000003D0"
-"0000000000000000000003D1"
-"0000000000000000000003D2"
-"0000000000000000000003D3"
-"0000000000000000000003D4"
-"0000000000000000000003D5"
-"0000000000000000000003D6"
-"0000000000000000000003D7"
-"0000000000000000000003D8"
-"0000000000000000000003D9"
-"0000000000000000000003DA"
-"0000000000000000000003DB"
-"0000000000000000000003DC"
-"0000000000000000000003DD"
-"0000000000000000000003DE"
-"0000000000000000000003DF"
-"0000000000000000000003E0"
-"0000000000000000000003E1"
-"0000000000000000000003E2"
-"0000000000000000000003E3"
-"0000000000000000000003E4"
-"0000000000000000000003E5"
-"0000000000000000000003E6"
-"0000000000000000000003E7"
-"0000000000000000000003E8"
-"0000000000000000000003E9"
-"0000000000000000000003EA"
-"0000000000000000000003EB"
-"0000000000000000000003EC"
-"0000000000000000000003ED"
-"0000000000000000000003EE"
-"0000000000000000000003EF"
-"0000000000000000000003F0"
-"0000000000000000000003F1"
-"0000000000000000000003F2"
-"0000000000000000000003F3"
-"0000000000000000000003F4"
-"0000000000000000000003F5"
-"0000000000000000000003F6"
-"0000000000000000000003F7"
-"0000000000000000000003F8"
-"0000000000000000000003F9"
-"0000000000000000000003FA"
-"0000000000000000000003FB"
-"0000000000000000000003FC"
-"0000000000000000000003FD"
-"0000000000000000000003FE"
-"0000000000000000000003FF"
-"000000000000000000000400"
-"000000000000000000000401"
-"000000000000000000000402"
-"000000000000000000000403"
-"000000000000000000000404"
-"000000000000000000000405"
-"000000000000000000000406"
-"000000000000000000000407"
-"000000000000000000000408"
-"000000000000000000000409"
-"00000000000000000000040A"
-"00000000000000000000040B"
-"00000000000000000000040C"
-"00000000000000000000040D"
-"00000000000000000000040E"
-"00000000000000000000040F"
-"000000000000000000000410"
-"000000000000000000000411"
-"000000000000000000000412"
-"000000000000000000000413"
-"000000000000000000000414"
-"000000000000000000000415"
-"000000000000000000000416"
-"000000000000000000000417"
-"000000000000000000000418"
-"000000000000000000000419"
-"00000000000000000000041A"
-"00000000000000000000041B"
-"00000000000000000000041C"
-"00000000000000000000041D"
-"00000000000000000000041E"
-"00000000000000000000041F"
-"000000000000000000000420"
-"000000000000000000000421"
-"000000000000000000000422"
-"000000000000000000000423"
-"000000000000000000000424"
-"000000000000000000000425"
-"000000000000000000000426"
-"000000000000000000000427"
-"000000000000000000000428"
-"000000000000000000000429"
-"00000000000000000000042A"
-"00000000000000000000042B"
-"00000000000000000000042C"
-"00000000000000000000042D"
-"00000000000000000000042E"
-"00000000000000000000042F"
-"000000000000000000000430"
-"000000000000000000000431"
-"000000000000000000000432"
-"000000000000000000000433"
-"000000000000000000000434"
-"000000000000000000000435"
-"000000000000000000000436"
-"000000000000000000000437"
-"000000000000000000000438"
-"000000000000000000000439"
-"00000000000000000000043A"
-"00000000000000000000043B"
-"00000000000000000000043C"
-"00000000000000000000043D"
-"00000000000000000000043E"
-"00000000000000000000043F"
-"000000000000000000000440"
-"000000000000000000000441"
-"000000000000000000000442"
-"000000000000000000000443"
-"000000000000000000000444"
-"000000000000000000000445"
-"000000000000000000000446"
-"000000000000000000000447"
-"000000000000000000000448"
-"000000000000000000000449"
-"00000000000000000000044A"
-"00000000000000000000044B"
-"00000000000000000000044C"
-"00000000000000000000044D"
-"00000000000000000000044E"
-"00000000000000000000044F"
-"000000000000000000000450"
-"000000000000000000000451"
-"000000000000000000000452"
-"000000000000000000000453"
-"000000000000000000000454"
-"000000000000000000000455"
-"000000000000000000000456"
-"000000000000000000000457"
-"000000000000000000000458"
-"000000000000000000000459"
-"00000000000000000000045A"
-"00000000000000000000045B"
-"00000000000000000000045C"
-"00000000000000000000045D"
-"00000000000000000000045E"
-"00000000000000000000045F"
-"000000000000000000000460"
-"000000000000000000000461"
-"000000000000000000000462"
-"000000000000000000000463"
-"000000000000000000000464"
-"000000000000000000000465"
-"000000000000000000000466"
-"000000000000000000000467"
-"000000000000000000000468"
-"000000000000000000000469"
-"00000000000000000000046A"
-"00000000000000000000046B"
-"00000000000000000000046C"
-"00000000000000000000046D"
-"00000000000000000000046E"
-"00000000000000000000046F"
-"000000000000000000000470"
-"000000000000000000000471"
-"000000000000000000000472"
-"000000000000000000000473"
-"000000000000000000000474"
-"000000000000000000000475"
-"000000000000000000000476"
-"000000000000000000000477"
-"000000000000000000000478"
-"000000000000000000000479"
-"00000000000000000000047A"
-"00000000000000000000047B"
-"00000000000000000000047C"
-"00000000000000000000047D"
-"00000000000000000000047E"
-"00000000000000000000047F"
-"000000000000000000000480"
-"000000000000000000000481"
-"000000000000000000000482"
-"000000000000000000000483"
-"000000000000000000000484"
-"000000000000000000000485"
-"000000000000000000000486"
-"000000000000000000000487"
-"000000000000000000000488"
-"000000000000000000000489"
-"00000000000000000000048A"
-"00000000000000000000048B"
-"00000000000000000000048C"
-"00000000000000000000048D"
-"00000000000000000000048E"
-"00000000000000000000048F"
-"000000000000000000000490"
-"000000000000000000000491"
-"000000000000000000000492"
-"000000000000000000000493"
-"000000000000000000000494"
-"000000000000000000000495"
-"000000000000000000000496"
-"000000000000000000000497"
-"000000000000000000000498"
-"000000000000000000000499"
-"00000000000000000000049A"
-"00000000000000000000049B"
-"00000000000000000000049C"
-"00000000000000000000049D"
-"00000000000000000000049E"
-"00000000000000000000049F"
-"0000000000000000000004A0"
-"0000000000000000000004A1"
-"0000000000000000000004A2"
-"0000000000000000000004A3"
-"0000000000000000000004A4"
-"0000000000000000000004A5"
-"0000000000000000000004A6"
-"0000000000000000000004A7"
-"0000000000000000000004A8"
-"0000000000000000000004A9"
-"0000000000000000000004AA"
-"0000000000000000000004AB"
-"0000000000000000000004AC"
-"0000000000000000000004AD"
-"0000000000000000000004AE"
-"0000000000000000000004AF"
-"0000000000000000000004B0"
-"0000000000000000000004B1"
-"0000000000000000000004B2"
-"0000000000000000000004B3"
-"0000000000000000000004B4"
-"0000000000000000000004B5"
-"0000000000000000000004B6"
-"0000000000000000000004B7"
-"0000000000000000000004B8"
-"0000000000000000000004B9"
-"0000000000000000000004BA"
-"0000000000000000000004BB"
-"0000000000000000000004BC"
-"0000000000000000000004BD"
-"0000000000000000000004BE"
-"0000000000000000000004BF"
-"0000000000000000000004C0"
-"0000000000000000000004C1"
-"0000000000000000000004C2"
-"0000000000000000000004C3"
-"0000000000000000000004C4"
-"0000000000000000000004C5"
-"0000000000000000000004C6"
-"0000000000000000000004C7"
-"0000000000000000000004C8"
-"0000000000000000000004C9"
-"0000000000000000000004CA"
-"0000000000000000000004CB"
-"0000000000000000000004CC"
-"0000000000000000000004CD"
-"0000000000000000000004CE"
-"0000000000000000000004CF"
-"0000000000000000000004D0"
-"0000000000000000000004D1"
-"0000000000000000000004D2"
-"0000000000000000000004D3"
-"0000000000000000000004D4"
-"0000000000000000000004D5"
-"0000000000000000000004D6"
-"0000000000000000000004D7"
-"0000000000000000000004D8"
-"0000000000000000000004D9"
-"0000000000000000000004DA"
-"0000000000000000000004DB"
-"0000000000000000000004DC"
-"0000000000000000000004DD"
-"0000000000000000000004DE"
-"0000000000000000000004DF"
-"0000000000000000000004E0"
-"0000000000000000000004E1"
-"0000000000000000000004E2"
-"0000000000000000000004E3"
-"0000000000000000000004E4"
-"0000000000000000000004E5"
-"0000000000000000000004E6"
-"0000000000000000000004E7"
-"0000000000000000000004E8"
-"0000000000000000000004E9"
-"0000000000000000000004EA"
-"0000000000000000000004EB"
-"0000000000000000000004EC"
-"0000000000000000000004ED"
-"0000000000000000000004EE"
-"0000000000000000000004EF"
-"0000000000000000000004F0"
-"0000000000000000000004F1"
-"0000000000000000000004F2"
-"0000000000000000000004F3"
-"0000000000000000000004F4"
-"0000000000000000000004F5"
-"0000000000000000000004F6"
-"0000000000000000000004F7"
-"0000000000000000000004F8"
-"0000000000000000000004F9"
-"0000000000000000000004FA"
-"0000000000000000000004FB"
-"0000000000000000000004FC"
-"0000000000000000000004FD"
-"0000000000000000000004FE"
-"0000000000000000000004FF"
-"000000000000000000000500"
-"000000000000000000000501"
-"000000000000000000000502"
-"000000000000000000000503"
-"000000000000000000000504"
-"000000000000000000000505"
-"000000000000000000000506"
-"000000000000000000000507"
-"000000000000000000000508"
-"000000000000000000000509"
-"00000000000000000000050A"
-"00000000000000000000050B"
-"00000000000000000000050C"
-"00000000000000000000050D"
-"00000000000000000000050E"
-"00000000000000000000050F"
-"000000000000000000000510"
-"000000000000000000000511"
-"000000000000000000000512"
-"000000000000000000000513"
-"000000000000000000000514"
-"000000000000000000000515"
-"000000000000000000000516"
-"000000000000000000000517"
-"000000000000000000000518"
-"000000000000000000000519"
-"00000000000000000000051A"
-"00000000000000000000051B"
-"00000000000000000000051C"
-"00000000000000000000051D"
-"00000000000000000000051E"
-"00000000000000000000051F"
-"000000000000000000000520"
-"000000000000000000000521"
-"000000000000000000000522"
-"000000000000000000000523"
-"000000000000000000000524"
-"000000000000000000000525"
-"000000000000000000000526"
-"000000000000000000000527"
-"000000000000000000000528"
-"000000000000000000000529"
-"00000000000000000000052A"
-"00000000000000000000052B"
-"00000000000000000000052C"
-"00000000000000000000052D"
-"00000000000000000000052E"
-"00000000000000000000052F"
-"000000000000000000000530"
-"000000000000000000000531"
-"000000000000000000000532"
-"000000000000000000000533"
-"000000000000000000000534"
-"000000000000000000000535"
-"000000000000000000000536"
-"000000000000000000000537"
-"000000000000000000000538"
-"000000000000000000000539"
-"00000000000000000000053A"
-"00000000000000000000053B"
-"00000000000000000000053C"
-"00000000000000000000053D"
-"00000000000000000000053E"
-"00000000000000000000053F"
-"000000000000000000000540"
-"000000000000000000000541"
-"000000000000000000000542"
-"000000000000000000000543"
-"000000000000000000000544"
-"000000000000000000000545"
-"000000000000000000000546"
-"000000000000000000000547"
-"000000000000000000000548"
-"000000000000000000000549"
-"00000000000000000000054A"
-"00000000000000000000054B"
-"00000000000000000000054C"
-"00000000000000000000054D"
-"00000000000000000000054E"
-"00000000000000000000054F"
-"000000000000000000000550"
-"000000000000000000000551"
-"000000000000000000000552"
-"000000000000000000000553"
-"000000000000000000000554"
-"000000000000000000000555"
-"000000000000000000000556"
-"000000000000000000000557"
-"000000000000000000000558"
-"000000000000000000000559"
-"00000000000000000000055A"
-"00000000000000000000055B"
-"00000000000000000000055C"
-"00000000000000000000055D"
-"00000000000000000000055E"
-"00000000000000000000055F"
-"000000000000000000000560"
-"000000000000000000000561"
-"000000000000000000000562"
-"000000000000000000000563"
-"000000000000000000000564"
-"000000000000000000000565"
-"000000000000000000000566"
-"000000000000000000000567"
-"000000000000000000000568"
-"000000000000000000000569"
-"00000000000000000000056A"
-"00000000000000000000056B"
-"00000000000000000000056C"
-"00000000000000000000056D"
-"00000000000000000000056E"
-"00000000000000000000056F"
-"000000000000000000000570"
-"000000000000000000000571"
-"000000000000000000000572"
-"000000000000000000000573"
-"000000000000000000000574"
-"000000000000000000000575"
-"000000000000000000000576"
-"000000000000000000000577"
-"000000000000000000000578"
-"000000000000000000000579"
-"00000000000000000000057A"
-"00000000000000000000057B"
-"00000000000000000000057C"
-"00000000000000000000057D"
-"00000000000000000000057E"
-"00000000000000000000057F"
-"000000000000000000000580"
-"000000000000000000000581"
-"000000000000000000000582"
-"000000000000000000000583"
-"000000000000000000000584"
-"000000000000000000000585"
-"000000000000000000000586"
-"000000000000000000000587"
-"000000000000000000000588"
-"000000000000000000000589"
-"00000000000000000000058A"
-"00000000000000000000058B"
-"00000000000000000000058C"
-"00000000000000000000058D"
-"00000000000000000000058E"
-"00000000000000000000058F"
-"000000000000000000000590"
-"000000000000000000000591"
-"000000000000000000000592"
-"000000000000000000000593"
-"000000000000000000000594"
-"000000000000000000000595"
-"000000000000000000000596"
-"000000000000000000000597"
-"000000000000000000000598"
-"000000000000000000000599"
-"00000000000000000000059A"
-"00000000000000000000059B"
-"00000000000000000000059C"
-"00000000000000000000059D"
-"00000000000000000000059E"
-"00000000000000000000059F"
-"0000000000000000000005A0"
-"0000000000000000000005A1"
-"0000000000000000000005A2"
-"0000000000000000000005A3"
-"0000000000000000000005A4"
-"0000000000000000000005A5"
-"0000000000000000000005A6"
-"0000000000000000000005A7"
-"0000000000000000000005A8"
-"0000000000000000000005A9"
-"0000000000000000000005AA"
-"0000000000000000000005AB"
-"0000000000000000000005AC"
-"0000000000000000000005AD"
-"0000000000000000000005AE"
-"0000000000000000000005AF"
-"0000000000000000000005B0"
-"0000000000000000000005B1"
-"0000000000000000000005B2"
-"0000000000000000000005B3"
-"0000000000000000000005B4"
-"0000000000000000000005B5"
-"0000000000000000000005B6"
-"0000000000000000000005B7"
-"0000000000000000000005B8"
-"0000000000000000000005B9"
-"0000000000000000000005BA"
-"0000000000000000000005BB"
-"0000000000000000000005BC"
-"0000000000000000000005BD"
-"0000000000000000000005BE"
-"0000000000000000000005BF"
-"0000000000000000000005C0"
-"0000000000000000000005C1"
-"0000000000000000000005C2"
-"0000000000000000000005C3"
-"0000000000000000000005C4"
-"0000000000000000000005C5"
-"0000000000000000000005C6"
-"0000000000000000000005C7"
-"0000000000000000000005C8"
-"0000000000000000000005C9"
-"0000000000000000000005CA"
-"0000000000000000000005CB"
-"0000000000000000000005CC"
-"0000000000000000000005CD"
-"0000000000000000000005CE"
-"0000000000000000000005CF"
-"0000000000000000000005D0"
-"0000000000000000000005D1"
-"0000000000000000000005D2"
-"0000000000000000000005D3"
-"0000000000000000000005D4"
-"0000000000000000000005D5"
-"0000000000000000000005D6"
-"0000000000000000000005D7"
-"0000000000000000000005D8"
-"0000000000000000000005D9"
-"0000000000000000000005DA"
-"0000000000000000000005DB"
-"0000000000000000000005DC"
-"0000000000000000000005DD"
-"0000000000000000000005DE"
-"0000000000000000000005DF"
-"0000000000000000000005E0"
-"0000000000000000000005E1"
-"0000000000000000000005E2"
-"0000000000000000000005E3"
-"0000000000000000000005E4"
-"0000000000000000000005E5"
-"0000000000000000000005E6"
-"0000000000000000000005E7"
-"000000010000000000000001"
-"000000010000000000000002"
-"000000010000000000000003"
-"000000010000000000000004"
-"000000010000000000000005"
-"000000010000000000000006"
-"000000010000000000000007"
-"000000010000000000000008"
-"000000010000000000000009"
-"00000001000000000000000A"
-"00000001000000000000000B"
-"00000001000000000000000C"
-"00000001000000000000000D"
-"00000001000000000000000E"
-"00000001000000000000000F"
-"000000010000000000000010"
-"000000010000000000000011"
-"000000010000000000000012"
-"000000010000000000000013"
-"000000010000000000000014"
-"000000010000000000000015"
-"000000010000000000000016"
-"000000010000000000000017"
-"000000010000000000000018"
-"000000010000000000000019"
-"00000001000000000000001A"
-"00000001000000000000001B"
-"00000001000000000000001C"
-"00000001000000000000001D"
-"00000001000000000000001E"
-"00000001000000000000001F"
-"000000010000000000000020"
-"000000010000000000000021"
-"000000010000000000000022"
-"000000010000000000000023"
-"000000010000000000000024"
-"000000010000000000000025"
-"000000010000000000000026"
-"000000010000000000000027"
-"000000010000000000000028"
-"000000010000000000000029"
-"00000001000000000000002A"
-"00000001000000000000002B"
-"00000001000000000000002C"
-"00000001000000000000002D"
-"00000001000000000000002E"
-"00000001000000000000002F"
-"000000010000000000000030"
-"000000010000000000000031"
-"000000010000000000000032"
-"000000010000000000000033"
-"000000010000000000000034"
-"000000010000000000000035"
-"000000010000000000000036"
-"000000010000000000000037"
-"000000010000000000000038"
-"000000010000000000000039"
-"00000001000000000000003A"
-"00000001000000000000003B"
-"00000001000000000000003C"
-"00000001000000000000003D"
-"00000001000000000000003E"
-"00000001000000000000003F"
-"000000010000000000000040"
-"000000010000000000000041"
-"000000010000000000000042"
-"000000010000000000000043"
-"000000010000000000000044"
-"000000010000000000000045"
-"000000010000000000000046"
-"000000010000000000000047"
-"000000010000000000000048"
-"000000010000000000000049"
-"00000001000000000000004A"
-"00000001000000000000004B"
-"00000001000000000000004C"
-"00000001000000000000004D"
-"00000001000000000000004E"
-"00000001000000000000004F"
-"000000010000000000000050"
-"000000010000000000000051"
-"000000010000000000000052"
-"000000010000000000000053"
-"000000010000000000000054"
-"000000010000000000000055"
-"000000010000000000000056"
-"000000010000000000000057"
-"000000010000000000000058"
-"000000010000000000000059"
-"00000001000000000000005A"
-"00000001000000000000005B"
-"00000001000000000000005C"
-"00000001000000000000005D"
-"00000001000000000000005E"
-"00000001000000000000005F"
-"000000010000000000000060"
-"000000010000000000000061"
-"000000010000000000000062"
-"000000010000000000000063"
-"000000010000000000000064"
-"000000010000000000000065"
-"000000010000000000000066"
-"000000010000000000000067"
-"000000010000000000000068"
-"000000010000000000000069"
-"00000001000000000000006A"
-"00000001000000000000006B"
-"00000001000000000000006C"
-"00000001000000000000006D"
-"00000001000000000000006E"
-"00000001000000000000006F"
-"000000010000000000000070"
-"000000010000000000000071"
-"000000010000000000000072"
-"000000010000000000000073"
-"000000010000000000000074"
-"000000010000000000000075"
-"000000010000000000000076"
-"000000010000000000000077"
-"000000010000000000000078"
-"000000010000000000000079"
-"00000001000000000000007A"
-"00000001000000000000007B"
-"00000001000000000000007C"
-"00000001000000000000007D"
-"00000001000000000000007E"
-"00000001000000000000007F"
-"000000010000000000000080"
-"000000010000000000000081"
-"000000010000000000000082"
-"000000010000000000000083"
-"000000010000000000000084"
-"000000010000000000000085"
-"000000010000000000000086"
-"000000010000000000000087"
-"000000010000000000000088"
-"000000010000000000000089"
-"00000001000000000000008A"
-"00000001000000000000008B"
-"00000001000000000000008C"
-"00000001000000000000008D"
-"00000001000000000000008E"
-"00000001000000000000008F"
-"000000010000000000000090"
-"000000010000000000000091"
-"000000010000000000000092"
-"000000010000000000000093"
-"000000010000000000000094"
-"000000010000000000000095"
-"000000010000000000000096"
-"000000010000000000000097"
-"000000010000000000000098"
-"000000010000000000000099"
-"00000001000000000000009A"
-"00000001000000000000009B"
-"00000001000000000000009C"
-"00000001000000000000009D"
-"00000001000000000000009E"
-"00000001000000000000009F"
-"0000000100000000000000A0"
-"0000000100000000000000A1"
-"0000000100000000000000A2"
-"0000000100000000000000A3"
-"0000000100000000000000A4"
-"0000000100000000000000A5"
-"0000000100000000000000A6"
-"0000000100000000000000A7"
-"0000000100000000000000A8"
-"0000000100000000000000A9"
-"0000000100000000000000AA"
-"0000000100000000000000AB"
-"0000000100000000000000AC"
-"0000000100000000000000AD"
-"0000000100000000000000AE"
-"0000000100000000000000AF"
-"0000000100000000000000B0"
-"0000000100000000000000B1"
-"0000000100000000000000B2"
-"0000000100000000000000B3"
-"0000000100000000000000B4"
-"0000000100000000000000B5"
-"0000000100000000000000B6"
-"0000000100000000000000B7"
-"0000000100000000000000B8"
-"0000000100000000000000B9"
-"0000000100000000000000BA"
-"0000000100000000000000BB"
-"0000000100000000000000BC"
-"0000000100000000000000BD"
-"0000000100000000000000BE"
-"0000000100000000000000BF"
-"0000000100000000000000C0"
-"0000000100000000000000C1"
-"0000000100000000000000C2"
-"0000000100000000000000C3"
-"0000000100000000000000C4"
-"0000000100000000000000C5"
-"0000000100000000000000C6"
-"0000000100000000000000C7"
-"0000000100000000000000C8"
-"0000000100000000000000C9"
-"0000000100000000000000CA"
-"0000000100000000000000CB"
-"0000000100000000000000CC"
-"0000000100000000000000CD"
-"0000000100000000000000CE"
-"0000000100000000000000CF"
-"0000000100000000000000D0"
-"0000000100000000000000D1"
-"0000000100000000000000D2"
-"0000000100000000000000D3"
-"0000000100000000000000D4"
-"0000000100000000000000D5"
-"0000000100000000000000D6"
-"0000000100000000000000D7"
-"0000000100000000000000D8"
-"0000000100000000000000D9"
-"0000000100000000000000DA"
-"0000000100000000000000DB"
-"0000000100000000000000DC"
-"0000000100000000000000DD"
-"0000000100000000000000DE"
-"0000000100000000000000DF"
-"0000000100000000000000E0"
-"0000000100000000000000E1"
-"0000000100000000000000E2"
-"0000000100000000000000E3"
-"0000000100000000000000E4"
-"0000000100000000000000E5"
-"0000000100000000000000E6"
-"0000000100000000000000E7"
-"0000000100000000000000E8"
-"0000000100000000000000E9"
-"0000000100000000000000EA"
-"0000000100000000000000EB"
-"0000000100000000000000EC"
-"0000000100000000000000ED"
-"0000000100000000000000EE"
-"0000000100000000000000EF"
-"0000000100000000000000F0"
-"0000000100000000000000F1"
-"0000000100000000000000F2"
-"0000000100000000000000F3"
-"0000000100000000000000F4"
-"0000000100000000000000F5"
-"0000000100000000000000F6"
-"0000000100000000000000F7"
-"0000000100000000000000F8"
-"0000000100000000000000F9"
-"0000000100000000000000FA"
-"0000000100000000000000FB"
-"0000000100000000000000FC"
-"0000000100000000000000FD"
-"0000000100000000000000FE"
-"0000000100000000000000FF"
-"000000010000000000000100"
-"000000010000000000000101"
-"000000010000000000000102"
-"000000010000000000000103"
-"000000010000000000000104"
-"000000010000000000000105"
-"000000010000000000000106"
-"000000010000000000000107"
-"000000010000000000000108"
-"000000010000000000000109"
-"00000001000000000000010A"
-"00000001000000000000010B"
-"00000001000000000000010C"
-"00000001000000000000010D"
-"00000001000000000000010E"
-"00000001000000000000010F"
-"000000010000000000000110"
-"000000010000000000000111"
-"000000010000000000000112"
-"000000010000000000000113"
-"000000010000000000000114"
-"000000010000000000000115"
-"000000010000000000000116"
-"000000010000000000000117"
-"000000010000000000000118"
-"000000010000000000000119"
-"00000001000000000000011A"
-"00000001000000000000011B"
-"00000001000000000000011C"
-"00000001000000000000011D"
-"00000001000000000000011E"
-"00000001000000000000011F"
-"000000010000000000000120"
-"000000010000000000000121"
-"000000010000000000000122"
-"000000010000000000000123"
-"000000010000000000000124"
-"000000010000000000000125"
-"000000010000000000000126"
-"000000010000000000000127"
-"000000010000000000000128"
-"000000010000000000000129"
-"00000001000000000000012A"
-"00000001000000000000012B"
-"00000001000000000000012C"
-"00000001000000000000012D"
-"00000001000000000000012E"
-"00000001000000000000012F"
-"000000010000000000000130"
-"000000010000000000000131"
-"000000010000000000000132"
-"000000010000000000000133"
-"000000010000000000000134"
-"000000010000000000000135"
-"000000010000000000000136"
-"000000010000000000000137"
-"000000010000000000000138"
-"000000010000000000000139"
-"00000001000000000000013A"
-"00000001000000000000013B"
-"00000001000000000000013C"
-"00000001000000000000013D"
-"00000001000000000000013E"
-"00000001000000000000013F"
-"000000010000000000000140"
-"000000010000000000000141"
-"000000010000000000000142"
-"000000010000000000000143"
-"000000010000000000000144"
-"000000010000000000000145"
-"000000010000000000000146"
-"000000010000000000000147"
-"000000010000000000000148"
-"000000010000000000000149"
-"00000001000000000000014A"
-"00000001000000000000014B"
-"00000001000000000000014C"
-"00000001000000000000014D"
-"00000001000000000000014E"
-"00000001000000000000014F"
-"000000010000000000000150"
-"000000010000000000000151"
-"000000010000000000000152"
-"000000010000000000000153"
-"000000010000000000000154"
-"000000010000000000000155"
-"000000010000000000000156"
-"000000010000000000000157"
-"000000010000000000000158"
-"000000010000000000000159"
-"00000001000000000000015A"
-"00000001000000000000015B"
-"00000001000000000000015C"
-"00000001000000000000015D"
-"00000001000000000000015E"
-"00000001000000000000015F"
-"000000010000000000000160"
-"000000010000000000000161"
-"000000010000000000000162"
-"000000010000000000000163"
-"000000010000000000000164"
-"000000010000000000000165"
-"000000010000000000000166"
-"000000010000000000000167"
-"000000010000000000000168"
-"000000010000000000000169"
-"00000001000000000000016A"
-"00000001000000000000016B"
-"00000001000000000000016C"
-"00000001000000000000016D"
-"00000001000000000000016E"
-"00000001000000000000016F"
-"000000010000000000000170"
-"000000010000000000000171"
-"000000010000000000000172"
-"000000010000000000000173"
-"000000010000000000000174"
-"000000010000000000000175"
-"000000010000000000000176"
-"000000010000000000000177"
-"000000010000000000000178"
-"000000010000000000000179"
-"00000001000000000000017A"
-"00000001000000000000017B"
-"00000001000000000000017C"
-"00000001000000000000017D"
-"00000001000000000000017E"
-"00000001000000000000017F"
-"000000010000000000000180"
-"000000010000000000000181"
-"000000010000000000000182"
-"000000010000000000000183"
-"000000010000000000000184"
-"000000010000000000000185"
-"000000010000000000000186"
-"000000010000000000000187"
-"000000010000000000000188"
-"000000010000000000000189"
-"00000001000000000000018A"
-"00000001000000000000018B"
-"00000001000000000000018C"
-"00000001000000000000018D"
-"00000001000000000000018E"
-"00000001000000000000018F"
-"000000010000000000000190"
-"000000010000000000000191"
-"000000010000000000000192"
-"000000010000000000000193"
-"000000010000000000000194"
-"000000010000000000000195"
-"000000010000000000000196"
-"000000010000000000000197"
-"000000010000000000000198"
-"000000010000000000000199"
-"00000001000000000000019A"
-"00000001000000000000019B"
-"00000001000000000000019C"
-"00000001000000000000019D"
-"00000001000000000000019E"
-"00000001000000000000019F"
-"0000000100000000000001A0"
-"0000000100000000000001A1"
-"0000000100000000000001A2"
-"0000000100000000000001A3"
-"0000000100000000000001A4"
-"0000000100000000000001A5"
-"0000000100000000000001A6"
-"0000000100000000000001A7"
-"0000000100000000000001A8"
-"0000000100000000000001A9"
-"0000000100000000000001AA"
-"0000000100000000000001AB"
-"0000000100000000000001AC"
-"0000000100000000000001AD"
-"0000000100000000000001AE"
-"0000000100000000000001AF"
-"0000000100000000000001B0"
-"0000000100000000000001B1"
-"0000000100000000000001B2"
-"0000000100000000000001B3"
-"0000000100000000000001B4"
-"0000000100000000000001B5"
-"0000000100000000000001B6"
-"0000000100000000000001B7"
-"0000000100000000000001B8"
-"0000000100000000000001B9"
-"0000000100000000000001BA"
-"0000000100000000000001BB"
-"0000000100000000000001BC"
-"0000000100000000000001BD"
-"0000000100000000000001BE"
-"0000000100000000000001BF"
-"0000000100000000000001C0"
-"0000000100000000000001C1"
-"0000000100000000000001C2"
-"0000000100000000000001C3"
-"0000000100000000000001C4"
-"0000000100000000000001C5"
-"0000000100000000000001C6"
-"0000000100000000000001C7"
-"0000000100000000000001C8"
-"0000000100000000000001C9"
-"0000000100000000000001CA"
-"0000000100000000000001CB"
-"0000000100000000000001CC"
-"0000000100000000000001CD"
-"0000000100000000000001CE"
-"0000000100000000000001CF"
-"0000000100000000000001D0"
-"0000000100000000000001D1"
-"0000000100000000000001D2"
-"0000000100000000000001D3"
-"0000000100000000000001D4"
-"0000000100000000000001D5"
-"0000000100000000000001D6"
-"0000000100000000000001D7"
-"0000000100000000000001D8"
-"0000000100000000000001D9"
-"0000000100000000000001DA"
-"0000000100000000000001DB"
-"0000000100000000000001DC"
-"0000000100000000000001DD"
-"0000000100000000000001DE"
-"0000000100000000000001DF"
-"0000000100000000000001E0"
-"0000000100000000000001E1"
-"0000000100000000000001E2"
-"0000000100000000000001E3"
-"0000000100000000000001E4"
-"0000000100000000000001E5"
-"0000000100000000000001E6"
-"0000000100000000000001E7"
-"0000000100000000000001E8"
-"0000000100000000000001E9"
-"0000000100000000000001EA"
-"0000000100000000000001EB"
-"0000000100000000000001EC"
-"0000000100000000000001ED"
-"0000000100000000000001EE"
-"0000000100000000000001EF"
-"0000000100000000000001F0"
-"0000000100000000000001F1"
-"0000000100000000000001F2"
-"0000000100000000000001F3"
-"0000000100000000000001F4"
-"0000000100000000000001F5"
-"0000000100000000000001F6"
-"0000000100000000000001F7"
-"0000000100000000000001F8"
-"0000000100000000000001F9"
-"0000000100000000000001FA"
-"0000000100000000000001FB"
-"0000000100000000000001FC"
-"0000000100000000000001FD"
-"0000000100000000000001FE"
-"0000000100000000000001FF"
-"000000010000000000000200"
-"000000010000000000000201"
-"000000010000000000000202"
-"000000010000000000000203"
-"000000010000000000000204"
-"000000010000000000000205"
-"000000010000000000000206"
-"000000010000000000000207"
-"000000010000000000000208"
-"000000010000000000000209"
-"00000001000000000000020A"
-"00000001000000000000020B"
-"00000001000000000000020C"
-"00000001000000000000020D"
-"00000001000000000000020E"
-"00000001000000000000020F"
-"000000010000000000000210"
-"000000010000000000000211"
-"000000010000000000000212"
-"000000010000000000000213"
-"000000010000000000000214"
-"000000010000000000000215"
-"000000010000000000000216"
-"000000010000000000000217"
-"000000010000000000000218"
-"000000010000000000000219"
-"00000001000000000000021A"
-"00000001000000000000021B"
-"00000001000000000000021C"
-"00000001000000000000021D"
-"00000001000000000000021E"
-"00000001000000000000021F"
-"000000010000000000000220"
-"000000010000000000000221"
-"000000010000000000000222"
-"000000010000000000000223"
-"000000010000000000000224"
-"000000010000000000000225"
-"000000010000000000000226"
-"000000010000000000000227"
-"000000010000000000000228"
-"000000010000000000000229"
-"00000001000000000000022A"
-"00000001000000000000022B"
-"00000001000000000000022C"
-"00000001000000000000022D"
-"00000001000000000000022E"
-"00000001000000000000022F"
-"000000010000000000000230"
-"000000010000000000000231"
-"000000010000000000000232"
-"000000010000000000000233"
-"000000010000000000000234"
-"000000010000000000000235"
-"000000010000000000000236"
-"000000010000000000000237"
-"000000010000000000000238"
-"000000010000000000000239"
-"00000001000000000000023A"
-"00000001000000000000023B"
-"00000001000000000000023C"
-"00000001000000000000023D"
-"00000001000000000000023E"
-"00000001000000000000023F"
-"000000010000000000000240"
-"000000010000000000000241"
-"000000010000000000000242"
-"000000010000000000000243"
-"000000010000000000000244"
-"000000010000000000000245"
-"000000010000000000000246"
-"000000010000000000000247"
-"000000010000000000000248"
-"000000010000000000000249"
-"00000001000000000000024A"
-"00000001000000000000024B"
-"00000001000000000000024C"
-"00000001000000000000024D"
-"00000001000000000000024E"
-"00000001000000000000024F"
-"000000010000000000000250"
-"000000010000000000000251"
-"000000010000000000000252"
-"000000010000000000000253"
-"000000010000000000000254"
-"000000010000000000000255"
-"000000010000000000000256"
-"000000010000000000000257"
-"000000010000000000000258"
-"000000010000000000000259"
-"00000001000000000000025A"
-"00000001000000000000025B"
-"00000001000000000000025C"
-"00000001000000000000025D"
-"00000001000000000000025E"
-"00000001000000000000025F"
-"000000010000000000000260"
-"000000010000000000000261"
-"000000010000000000000262"
-"000000010000000000000263"
-"000000010000000000000264"
-"000000010000000000000265"
-"000000010000000000000266"
-"000000010000000000000267"
-"000000010000000000000268"
-"000000010000000000000269"
-"00000001000000000000026A"
-"00000001000000000000026B"
-"00000001000000000000026C"
-"00000001000000000000026D"
-"00000001000000000000026E"
-"00000001000000000000026F"
-"000000010000000000000270"
-"000000010000000000000271"
-"000000010000000000000272"
-"000000010000000000000273"
-"000000010000000000000274"
-"000000010000000000000275"
-"000000010000000000000276"
-"000000010000000000000277"
-"000000010000000000000278"
-"000000010000000000000279"
-"00000001000000000000027A"
-"00000001000000000000027B"
-"00000001000000000000027C"
-"00000001000000000000027D"
-"00000001000000000000027E"
-"00000001000000000000027F"
-"000000010000000000000280"
-"000000010000000000000281"
-"000000010000000000000282"
-"000000010000000000000283"
-"000000010000000000000284"
-"000000010000000000000285"
-"000000010000000000000286"
-"000000010000000000000287"
-"000000010000000000000288"
-"000000010000000000000289"
-"00000001000000000000028A"
-"00000001000000000000028B"
-"00000001000000000000028C"
-"00000001000000000000028D"
-"00000001000000000000028E"
-"00000001000000000000028F"
-"000000010000000000000290"
-"000000010000000000000291"
-"000000010000000000000292"
-"000000010000000000000293"
-"000000010000000000000294"
-"000000010000000000000295"
-"000000010000000000000296"
-"000000010000000000000297"
-"000000010000000000000298"
-"000000010000000000000299"
-"00000001000000000000029A"
-"00000001000000000000029B"
-"00000001000000000000029C"
-"00000001000000000000029D"
-"00000001000000000000029E"
-"00000001000000000000029F"
-"0000000100000000000002A0"
-"0000000100000000000002A1"
-"0000000100000000000002A2"
-"0000000100000000000002A3"
-"0000000100000000000002A4"
-"0000000100000000000002A5"
-"0000000100000000000002A6"
-"0000000100000000000002A7"
-"0000000100000000000002A8"
-"0000000100000000000002A9"
-"0000000100000000000002AA"
-"0000000100000000000002AB"
-"0000000100000000000002AC"
-"0000000100000000000002AD"
-"0000000100000000000002AE"
-"0000000100000000000002AF"
-"0000000100000000000002B0"
-"0000000100000000000002B1"
-"0000000100000000000002B2"
-"0000000100000000000002B3"
-"0000000100000000000002B4"
-"0000000100000000000002B5"
-"0000000100000000000002B6"
-"0000000100000000000002B7"
-"0000000100000000000002B8"
-"0000000100000000000002B9"
-"0000000100000000000002BA"
-"0000000100000000000002BB"
-"0000000100000000000002BC"
-"0000000100000000000002BD"
-"0000000100000000000002BE"
-"0000000100000000000002BF"
-"0000000100000000000002C0"
-"0000000100000000000002C1"
-"0000000100000000000002C2"
-"0000000100000000000002C3"
-"0000000100000000000002C4"
-"0000000100000000000002C5"
-"0000000100000000000002C6"
-"0000000100000000000002C7"
-"0000000100000000000002C8"
-"0000000100000000000002C9"
-"0000000100000000000002CA"
-"0000000100000000000002CB"
-"0000000100000000000002CC"
-"0000000100000000000002CD"
-"0000000100000000000002CE"
-"0000000100000000000002CF"
-"0000000100000000000002D0"
-"0000000100000000000002D1"
-"0000000100000000000002D2"
-"0000000100000000000002D3"
-"0000000100000000000002D4"
-"0000000100000000000002D5"
-"0000000100000000000002D6"
-"0000000100000000000002D7"
-"0000000100000000000002D8"
-"0000000100000000000002D9"
-"0000000100000000000002DA"
-"0000000100000000000002DB"
-"0000000100000000000002DC"
-"0000000100000000000002DD"
-"0000000100000000000002DE"
-"0000000100000000000002DF"
-"0000000100000000000002E0"
-"0000000100000000000002E1"
-"0000000100000000000002E2"
-"0000000100000000000002E3"
-"0000000100000000000002E4"
-"0000000100000000000002E5"
-"0000000100000000000002E6"
-"0000000100000000000002E7"
-"0000000100000000000002E8"
-"0000000100000000000002E9"
-"0000000100000000000002EA"
-"0000000100000000000002EB"
-"0000000100000000000002EC"
-"0000000100000000000002ED"
-"0000000100000000000002EE"
-"0000000100000000000002EF"
-"0000000100000000000002F0"
-"0000000100000000000002F1"
-"0000000100000000000002F2"
-"0000000100000000000002F3"
-"0000000100000000000002F4"
-"0000000100000000000002F5"
-"0000000100000000000002F6"
-"0000000100000000000002F7"
-"0000000100000000000002F8"
-"0000000100000000000002F9"
-"0000000100000000000002FA"
-"0000000100000000000002FB"
-"0000000100000000000002FC"
-"0000000100000000000002FD"
-"0000000100000000000002FE"
-"0000000100000000000002FF"
-"000000010000000000000300"
-"000000010000000000000301"
-"000000010000000000000302"
-"000000010000000000000303"
-"000000010000000000000304"
-"000000010000000000000305"
-"000000010000000000000306"
-"000000010000000000000307"
-"000000010000000000000308"
-"000000010000000000000309"
-"00000001000000000000030A"
-"00000001000000000000030B"
-"00000001000000000000030C"
-"00000001000000000000030D"
-"00000001000000000000030E"
-"00000001000000000000030F"
-"000000010000000000000310"
-"000000010000000000000311"
-"000000010000000000000312"
-"000000010000000000000313"
-"000000010000000000000314"
-"000000010000000000000315"
-"000000010000000000000316"
-"000000010000000000000317"
-"000000010000000000000318"
-"000000010000000000000319"
-"00000001000000000000031A"
-"00000001000000000000031B"
-"00000001000000000000031C"
-"00000001000000000000031D"
-"00000001000000000000031E"
-"00000001000000000000031F"
-"000000010000000000000320"
-"000000010000000000000321"
-"000000010000000000000322"
-"000000010000000000000323"
-"000000010000000000000324"
-"000000010000000000000325"
-"000000010000000000000326"
-"000000010000000000000327"
-"000000010000000000000328"
-"000000010000000000000329"
-"00000001000000000000032A"
-"00000001000000000000032B"
-"00000001000000000000032C"
-"00000001000000000000032D"
-"00000001000000000000032E"
-"00000001000000000000032F"
-"000000010000000000000330"
-"000000010000000000000331"
-"000000010000000000000332"
-"000000010000000000000333"
-"000000010000000000000334"
-"000000010000000000000335"
-"000000010000000000000336"
-"000000010000000000000337"
-"000000010000000000000338"
-"000000010000000000000339"
-"00000001000000000000033A"
-"00000001000000000000033B"
-"00000001000000000000033C"
-"00000001000000000000033D"
-"00000001000000000000033E"
-"00000001000000000000033F"
-"000000010000000000000340"
-"000000010000000000000341"
-"000000010000000000000342"
-"000000010000000000000343"
-"000000010000000000000344"
-"000000010000000000000345"
-"000000010000000000000346"
-"000000010000000000000347"
-"000000010000000000000348"
-"000000010000000000000349"
-"00000001000000000000034A"
-"00000001000000000000034B"
-"00000001000000000000034C"
-"00000001000000000000034D"
-"00000001000000000000034E"
-"00000001000000000000034F"
-"000000010000000000000350"
-"000000010000000000000351"
-"000000010000000000000352"
-"000000010000000000000353"
-"000000010000000000000354"
-"000000010000000000000355"
-"000000010000000000000356"
-"000000010000000000000357"
-"000000010000000000000358"
-"000000010000000000000359"
-"00000001000000000000035A"
-"00000001000000000000035B"
-"00000001000000000000035C"
-"00000001000000000000035D"
-"00000001000000000000035E"
-"00000001000000000000035F"
-"000000010000000000000360"
-"000000010000000000000361"
-"000000010000000000000362"
-"000000010000000000000363"
-"000000010000000000000364"
-"000000010000000000000365"
-"000000010000000000000366"
-"000000010000000000000367"
-"000000010000000000000368"
-"000000010000000000000369"
-"00000001000000000000036A"
-"00000001000000000000036B"
-"00000001000000000000036C"
-"00000001000000000000036D"
-"00000001000000000000036E"
-"00000001000000000000036F"
-"000000010000000000000370"
-"000000010000000000000371"
-"000000010000000000000372"
-"000000010000000000000373"
-"000000010000000000000374"
-"000000010000000000000375"
-"000000010000000000000376"
-"000000010000000000000377"
-"000000010000000000000378"
-"000000010000000000000379"
-"00000001000000000000037A"
-"00000001000000000000037B"
-"00000001000000000000037C"
-"00000001000000000000037D"
-"00000001000000000000037E"
-"00000001000000000000037F"
-"000000010000000000000380"
-"000000010000000000000381"
-"000000010000000000000382"
-"000000010000000000000383"
-"000000010000000000000384"
-"000000010000000000000385"
-"000000010000000000000386"
-"000000010000000000000387"
-"000000010000000000000388"
-"000000010000000000000389"
-"00000001000000000000038A"
-"00000001000000000000038B"
-"00000001000000000000038C"
-"00000001000000000000038D"
-"00000001000000000000038E"
-"00000001000000000000038F"
-"000000010000000000000390"
-"000000010000000000000391"
-"000000010000000000000392"
-"000000010000000000000393"
-"000000010000000000000394"
-"000000010000000000000395"
-"000000010000000000000396"
-"000000010000000000000397"
-"000000010000000000000398"
-"000000010000000000000399"
-"00000001000000000000039A"
-"00000001000000000000039B"
-"00000001000000000000039C"
-"00000001000000000000039D"
-"00000001000000000000039E"
-"00000001000000000000039F"
-"0000000100000000000003A0"
-"0000000100000000000003A1"
-"0000000100000000000003A2"
-"0000000100000000000003A3"
-"0000000100000000000003A4"
-"0000000100000000000003A5"
-"0000000100000000000003A6"
-"0000000100000000000003A7"
-"0000000100000000000003A8"
-"0000000100000000000003A9"
-"0000000100000000000003AA"
-"0000000100000000000003AB"
-"0000000100000000000003AC"
-"0000000100000000000003AD"
-"0000000100000000000003AE"
-"0000000100000000000003AF"
-"0000000100000000000003B0"
-"0000000100000000000003B1"
-"0000000100000000000003B2"
-"0000000100000000000003B3"
-"0000000100000000000003B4"
-"0000000100000000000003B5"
-"0000000100000000000003B6"
-"0000000100000000000003B7"
-"0000000100000000000003B8"
-"0000000100000000000003B9"
-"0000000100000000000003BA"
-"0000000100000000000003BB"
-"0000000100000000000003BC"
-"0000000100000000000003BD"
-"0000000100000000000003BE"
-"0000000100000000000003BF"
-"0000000100000000000003C0"
-"0000000100000000000003C1"
-"0000000100000000000003C2"
-"0000000100000000000003C3"
-"0000000100000000000003C4"
-"0000000100000000000003C5"
-"0000000100000000000003C6"
-"0000000100000000000003C7"
-"0000000100000000000003C8"
-"0000000100000000000003C9"
-"0000000100000000000003CA"
-"0000000100000000000003CB"
-"0000000100000000000003CC"
-"0000000100000000000003CD"
-"0000000100000000000003CE"
-"0000000100000000000003CF"
-"0000000100000000000003D0"
-"0000000100000000000003D1"
-"0000000100000000000003D2"
-"0000000100000000000003D3"
-"0000000100000000000003D4"
-"0000000100000000000003D5"
-"0000000100000000000003D6"
-"0000000100000000000003D7"
-"0000000100000000000003D8"
-"0000000100000000000003D9"
-"0000000100000000000003DA"
-"0000000100000000000003DB"
-"0000000100000000000003DC"
-"0000000100000000000003DD"
-"0000000100000000000003DE"
-"0000000100000000000003DF"
-"0000000100000000000003E0"
-"0000000100000000000003E1"
-"0000000100000000000003E2"
-"0000000100000000000003E3"
-"0000000100000000000003E4"
-"0000000100000000000003E5"
-"0000000100000000000003E6"
-"0000000100000000000003E7"
-"0000000100000000000003E8"
-"0000000100000000000003E9"
-"0000000100000000000003EA"
-"0000000100000000000003EB"
-"0000000100000000000003EC"
-"0000000100000000000003ED"
-"0000000100000000000003EE"
-"0000000100000000000003EF"
-"0000000100000000000003F0"
-"0000000100000000000003F1"
-"0000000100000000000003F2"
-"0000000100000000000003F3"
-"0000000100000000000003F4"
-"0000000100000000000003F5"
-"0000000100000000000003F6"
-"0000000100000000000003F7"
-"0000000100000000000003F8"
-"0000000100000000000003F9"
-"0000000100000000000003FA"
-"0000000100000000000003FB"
-"0000000100000000000003FC"
-"0000000100000000000003FD"
-"0000000100000000000003FE"
-"0000000100000000000003FF"
-"000000010000000000000400"
-"000000010000000000000401"
-"000000010000000000000402"
-"000000010000000000000403"
-"000000010000000000000404"
-"000000010000000000000405"
-"000000010000000000000406"
-"000000010000000000000407"
-"000000010000000000000408"
-"000000010000000000000409"
-"00000001000000000000040A"
-"00000001000000000000040B"
-"00000001000000000000040C"
-"00000001000000000000040D"
-"00000001000000000000040E"
-"00000001000000000000040F"
-"000000010000000000000410"
-"000000010000000000000411"
-"000000010000000000000412"
-"000000010000000000000413"
-"000000010000000000000414"
-"000000010000000000000415"
-"000000010000000000000416"
-"000000010000000000000417"
-"000000010000000000000418"
-"000000010000000000000419"
-"00000001000000000000041A"
-"00000001000000000000041B"
-"00000001000000000000041C"
-"00000001000000000000041D"
-"00000001000000000000041E"
-"00000001000000000000041F"
-"000000010000000000000420"
-"000000010000000000000421"
-"000000010000000000000422"
-"000000010000000000000423"
-"000000010000000000000424"
-"000000010000000000000425"
-"000000010000000000000426"
-"000000010000000000000427"
-"000000010000000000000428"
-"000000010000000000000429"
-"00000001000000000000042A"
-"00000001000000000000042B"
-"00000001000000000000042C"
-"00000001000000000000042D"
-"00000001000000000000042E"
-"00000001000000000000042F"
-"000000010000000000000430"
-"000000010000000000000431"
-"000000010000000000000432"
-"000000010000000000000433"
-"000000010000000000000434"
-"000000010000000000000435"
-"000000010000000000000436"
-"000000010000000000000437"
-"000000010000000000000438"
-"000000010000000000000439"
-"00000001000000000000043A"
-"00000001000000000000043B"
-"00000001000000000000043C"
-"00000001000000000000043D"
-"00000001000000000000043E"
-"00000001000000000000043F"
-"000000010000000000000440"
-"000000010000000000000441"
-"000000010000000000000442"
-"000000010000000000000443"
-"000000010000000000000444"
-"000000010000000000000445"
-"000000010000000000000446"
-"000000010000000000000447"
-"000000010000000000000448"
-"000000010000000000000449"
-"00000001000000000000044A"
-"00000001000000000000044B"
-"00000001000000000000044C"
-"00000001000000000000044D"
-"00000001000000000000044E"
-"00000001000000000000044F"
-"000000010000000000000450"
-"000000010000000000000451"
-"000000010000000000000452"
-"000000010000000000000453"
-"000000010000000000000454"
-"000000010000000000000455"
-"000000010000000000000456"
-"000000010000000000000457"
-"000000010000000000000458"
-"000000010000000000000459"
-"00000001000000000000045A"
-"00000001000000000000045B"
-"00000001000000000000045C"
-"00000001000000000000045D"
-"00000001000000000000045E"
-"00000001000000000000045F"
-"000000010000000000000460"
-"000000010000000000000461"
-"000000010000000000000462"
-"000000010000000000000463"
-"000000010000000000000464"
-"000000010000000000000465"
-"000000010000000000000466"
-"000000010000000000000467"
-"000000010000000000000468"
-"000000010000000000000469"
-"00000001000000000000046A"
-"00000001000000000000046B"
-"00000001000000000000046C"
-"00000001000000000000046D"
-"00000001000000000000046E"
-"00000001000000000000046F"
-"000000010000000000000470"
-"000000010000000000000471"
-"000000010000000000000472"
-"000000010000000000000473"
-"000000010000000000000474"
-"000000010000000000000475"
-"000000010000000000000476"
-"000000010000000000000477"
-"000000010000000000000478"
-"000000010000000000000479"
-"00000001000000000000047A"
-"00000001000000000000047B"
-"00000001000000000000047C"
-"00000001000000000000047D"
-"00000001000000000000047E"
-"00000001000000000000047F"
-"000000010000000000000480"
-"000000010000000000000481"
-"000000010000000000000482"
-"000000010000000000000483"
-"000000010000000000000484"
-"000000010000000000000485"
-"000000010000000000000486"
-"000000010000000000000487"
-"000000010000000000000488"
-"000000010000000000000489"
-"00000001000000000000048A"
-"00000001000000000000048B"
-"00000001000000000000048C"
-"00000001000000000000048D"
-"00000001000000000000048E"
-"00000001000000000000048F"
-"000000010000000000000490"
-"000000010000000000000491"
-"000000010000000000000492"
-"000000010000000000000493"
-"000000010000000000000494"
-"000000010000000000000495"
-"000000010000000000000496"
-"000000010000000000000497"
-"000000010000000000000498"
-"000000010000000000000499"
-"00000001000000000000049A"
-"00000001000000000000049B"
-"00000001000000000000049C"
-"00000001000000000000049D"
-"00000001000000000000049E"
-"00000001000000000000049F"
-"0000000100000000000004A0"
-"0000000100000000000004A1"
-"0000000100000000000004A2"
-"0000000100000000000004A3"
-"0000000100000000000004A4"
-"0000000100000000000004A5"
-"0000000100000000000004A6"
-"0000000100000000000004A7"
-"0000000100000000000004A8"
-"0000000100000000000004A9"
-"0000000100000000000004AA"
-"0000000100000000000004AB"
-"0000000100000000000004AC"
-"0000000100000000000004AD"
-"0000000100000000000004AE"
-"0000000100000000000004AF"
-"0000000100000000000004B0"
-"0000000100000000000004B1"
-"0000000100000000000004B2"
-"0000000100000000000004B3"
-"0000000100000000000004B4"
-"0000000100000000000004B5"
-"0000000100000000000004B6"
-"0000000100000000000004B7"
-"0000000100000000000004B8"
-"0000000100000000000004B9"
-"0000000100000000000004BA"
-"0000000100000000000004BB"
-"0000000100000000000004BC"
-"0000000100000000000004BD"
-"0000000100000000000004BE"
-"0000000100000000000004BF"
-"0000000100000000000004C0"
-"0000000100000000000004C1"
-"0000000100000000000004C2"
-"0000000100000000000004C3"
-"0000000100000000000004C4"
-"0000000100000000000004C5"
-"0000000100000000000004C6"
-"0000000100000000000004C7"
-"0000000100000000000004C8"
-"0000000100000000000004C9"
-"0000000100000000000004CA"
-"0000000100000000000004CB"
-"0000000100000000000004CC"
-"0000000100000000000004CD"
-"0000000100000000000004CE"
-"0000000100000000000004CF"
-"0000000100000000000004D0"
-"0000000100000000000004D1"
-"0000000100000000000004D2"
-"0000000100000000000004D3"
-"0000000100000000000004D4"
-"0000000100000000000004D5"
-"0000000100000000000004D6"
-"0000000100000000000004D7"
-"0000000100000000000004D8"
-"0000000100000000000004D9"
-"0000000100000000000004DA"
-"0000000100000000000004DB"
-"0000000100000000000004DC"
-"0000000100000000000004DD"
-"0000000100000000000004DE"
-"0000000100000000000004DF"
-"0000000100000000000004E0"
-"0000000100000000000004E1"
-"0000000100000000000004E2"
-"0000000100000000000004E3"
-"0000000100000000000004E4"
-"0000000100000000000004E5"
-"0000000100000000000004E6"
-"0000000100000000000004E7"
-"0000000100000000000004E8"
-"0000000100000000000004E9"
-"0000000100000000000004EA"
-"0000000100000000000004EB"
-"0000000100000000000004EC"
-"0000000100000000000004ED"
-"0000000100000000000004EE"
-"0000000100000000000004EF"
-"0000000100000000000004F0"
-"0000000100000000000004F1"
-"0000000100000000000004F2"
-"0000000100000000000004F3"
-"0000000100000000000004F4"
-"0000000100000000000004F5"
-"0000000100000000000004F6"
-"0000000100000000000004F7"
-"0000000100000000000004F8"
-"0000000100000000000004F9"
-"0000000100000000000004FA"
-"0000000100000000000004FB"
-"0000000100000000000004FC"
-"0000000100000000000004FD"
-"0000000100000000000004FE"
-"0000000100000000000004FF"
-"000000010000000000000500"
-"000000010000000000000501"
-"000000010000000000000502"
-"000000010000000000000503"
-"000000010000000000000504"
-"000000010000000000000505"
-"000000010000000000000506"
-"000000010000000000000507"
-"000000010000000000000508"
-"000000010000000000000509"
-"00000001000000000000050A"
-"00000001000000000000050B"
-"00000001000000000000050C"
-"00000001000000000000050D"
-"00000001000000000000050E"
-"00000001000000000000050F"
-"000000010000000000000510"
-"000000010000000000000511"
-"000000010000000000000512"
-"000000010000000000000513"
-"000000010000000000000514"
-"000000010000000000000515"
-"000000010000000000000516"
-"000000010000000000000517"
-"000000010000000000000518"
-"000000010000000000000519"
-"00000001000000000000051A"
-"00000001000000000000051B"
-"00000001000000000000051C"
-"00000001000000000000051D"
-"00000001000000000000051E"
-"00000001000000000000051F"
-"000000010000000000000520"
-"000000010000000000000521"
-"000000010000000000000522"
-"000000010000000000000523"
-"000000010000000000000524"
-"000000010000000000000525"
-"000000010000000000000526"
-"000000010000000000000527"
-"000000010000000000000528"
-"000000010000000000000529"
-"00000001000000000000052A"
-"00000001000000000000052B"
-"00000001000000000000052C"
-"00000001000000000000052D"
-"00000001000000000000052E"
-"00000001000000000000052F"
-"000000010000000000000530"
-"000000010000000000000531"
-"000000010000000000000532"
-"000000010000000000000533"
-"000000010000000000000534"
-"000000010000000000000535"
-"000000010000000000000536"
-"000000010000000000000537"
-"000000010000000000000538"
-"000000010000000000000539"
-"00000001000000000000053A"
-"00000001000000000000053B"
-"00000001000000000000053C"
-"00000001000000000000053D"
-"00000001000000000000053E"
-"00000001000000000000053F"
-"000000010000000000000540"
-"000000010000000000000541"
-"000000010000000000000542"
-"000000010000000000000543"
-"000000010000000000000544"
-"000000010000000000000545"
-"000000010000000000000546"
-"000000010000000000000547"
-"000000010000000000000548"
-"000000010000000000000549"
-"00000001000000000000054A"
-"00000001000000000000054B"
-"00000001000000000000054C"
-"00000001000000000000054D"
-"00000001000000000000054E"
-"00000001000000000000054F"
-"000000010000000000000550"
-"000000010000000000000551"
-"000000010000000000000552"
-"000000010000000000000553"
-"000000010000000000000554"
-"000000010000000000000555"
-"000000010000000000000556"
-"000000010000000000000557"
-"000000010000000000000558"
-"000000010000000000000559"
-"00000001000000000000055A"
-"00000001000000000000055B"
-"00000001000000000000055C"
-"00000001000000000000055D"
-"00000001000000000000055E"
-"00000001000000000000055F"
-"000000010000000000000560"
-"000000010000000000000561"
-"000000010000000000000562"
-"000000010000000000000563"
-"000000010000000000000564"
-"000000010000000000000565"
-"000000010000000000000566"
-"000000010000000000000567"
-"000000010000000000000568"
-"000000010000000000000569"
-"00000001000000000000056A"
-"00000001000000000000056B"
-"00000001000000000000056C"
-"00000001000000000000056D"
-"00000001000000000000056E"
-"00000001000000000000056F"
-"000000010000000000000570"
-"000000010000000000000571"
-"000000010000000000000572"
-"000000010000000000000573"
-"000000010000000000000574"
-"000000010000000000000575"
-"000000010000000000000576"
-"000000010000000000000577"
-"000000010000000000000578"
-"000000010000000000000579"
-"00000001000000000000057A"
-"00000001000000000000057B"
-"00000001000000000000057C"
-"00000001000000000000057D"
-"00000001000000000000057E"
-"00000001000000000000057F"
-"000000010000000000000580"
-"000000010000000000000581"
-"000000010000000000000582"
-"000000010000000000000583"
-"000000010000000000000584"
-"000000010000000000000585"
-"000000010000000000000586"
-"000000010000000000000587"
-"000000010000000000000588"
-"000000010000000000000589"
-"00000001000000000000058A"
-"00000001000000000000058B"
-"00000001000000000000058C"
-"00000001000000000000058D"
-"00000001000000000000058E"
-"00000001000000000000058F"
-"000000010000000000000590"
-"000000010000000000000591"
-"000000010000000000000592"
-"000000010000000000000593"
-"000000010000000000000594"
-"000000010000000000000595"
-"000000010000000000000596"
-"000000010000000000000597"
-"000000010000000000000598"
-"000000010000000000000599"
-"00000001000000000000059A"
-"00000001000000000000059B"
-"00000001000000000000059C"
-"00000001000000000000059D"
-"00000001000000000000059E"
-"00000001000000000000059F"
-"0000000100000000000005A0"
-"0000000100000000000005A1"
-"0000000100000000000005A2"
-"0000000100000000000005A3"
-"0000000100000000000005A4"
-"0000000100000000000005A5"
-"0000000100000000000005A6"
-"0000000100000000000005A7"
-"0000000100000000000005A8"
-"0000000100000000000005A9"
-"0000000100000000000005AA"
-"0000000100000000000005AB"
-"0000000100000000000005AC"
-"0000000100000000000005AD"
-"0000000100000000000005AE"
-"0000000100000000000005AF"
-"0000000100000000000005B0"
-"0000000100000000000005B1"
-"0000000100000000000005B2"
-"0000000100000000000005B3"
-"0000000100000000000005B4"
-"0000000100000000000005B5"
-"0000000100000000000005B6"
-"0000000100000000000005B7"
-"0000000100000000000005B8"
-"0000000100000000000005B9"
-"0000000100000000000005BA"
-"0000000100000000000005BB"
-"0000000100000000000005BC"
-"0000000100000000000005BD"
-"0000000100000000000005BE"
-"0000000100000000000005BF"
-"0000000100000000000005C0"
-"0000000100000000000005C1"
-"0000000100000000000005C2"
-"0000000100000000000005C3"
-"0000000100000000000005C4"
-"0000000100000000000005C5"
-"0000000100000000000005C6"
-"0000000100000000000005C7"
-"0000000100000000000005C8"
-"0000000100000000000005C9"
-"0000000100000000000005CA"
-"0000000100000000000005CB"
-"0000000100000000000005CC"
-"0000000100000000000005CD"
-"0000000100000000000005CE"
-"0000000100000000000005CF"
-"0000000100000000000005D0"
-"0000000100000000000005D1"
-"0000000100000000000005D2"
-"0000000100000000000005D3"
-"0000000100000000000005D4"
-"0000000100000000000005D5"
-"0000000100000000000005D6"
-"0000000100000000000005D7"
-"0000000100000000000005D8"
-"0000000100000000000005D9"
-"0000000100000000000005DA"
-"0000000100000000000005DB"
-"0000000100000000000005DC"
-"0000000100000000000005DD"
-"0000000100000000000005DE"
-"0000000100000000000005DF"
-"0000000100000000000005E0"
-"0000000100000000000005E1"
-"0000000100000000000005E2"
-"0000000100000000000005E3"
-"0000000100000000000005E4"
-"0000000100000000000005E5"
-"0000000100000000000005E6"
-"0000000100000000000005E7"
-"0000000100000000000005E8"
-"0000000100000000000005E9"
-"0000000100000000000005EA"
-"0000000100000000000005EB"
-"0000000100000000000005EC"
-"0000000100000000000005ED"
-"0000000100000000000005EE"
-"0000000100000000000005EF"
-"0000000100000000000005F0"
-"0000000100000000000005F1"
-"0000000100000000000005F2"
-"0000000100000000000005F3"
-"0000000100000000000005F4"
-"000000020000000000000001"
-"000000020000000000000002"
-"000000020000000000000003"
-"000000020000000000000004"
-"000000020000000000000005"
-"000000020000000000000006"
-"000000020000000000000007"
-"000000020000000000000008"
-"000000020000000000000009"
-"00000002000000000000000A"
-"00000002000000000000000B"
-"00000002000000000000000C"
-"00000002000000000000000D"
-"00000002000000000000000E"
-"00000002000000000000000F"
-"000000020000000000000010"
-"000000020000000000000011"
-"000000020000000000000012"
-"000000020000000000000013"
-"000000020000000000000014"
-"000000020000000000000015"
-"000000020000000000000016"
-"000000020000000000000017"
-"000000020000000000000018"
-"000000020000000000000019"
-"00000002000000000000001A"
-"00000002000000000000001B"
-"00000002000000000000001C"
-"00000002000000000000001D"
-"00000002000000000000001E"
-"00000002000000000000001F"
-"000000020000000000000020"
-"000000020000000000000021"
-"000000020000000000000022"
-"000000020000000000000023"
-"000000020000000000000024"
-"000000020000000000000025"
-"000000020000000000000026"
-"000000020000000000000027"
-"000000020000000000000028"
-"000000020000000000000029"
-"00000002000000000000002A"
-"00000002000000000000002B"
-"00000002000000000000002C"
-"00000002000000000000002D"
-"00000002000000000000002E"
-"00000002000000000000002F"
-"000000020000000000000030"
-"000000020000000000000031"
-"000000020000000000000032"
-"000000020000000000000033"
-"000000020000000000000034"
-"000000020000000000000035"
-"000000020000000000000036"
-"000000020000000000000037"
-"000000020000000000000038"
-"000000020000000000000039"
-"00000002000000000000003A"
-"00000002000000000000003B"
-"00000002000000000000003C"
-"00000002000000000000003D"
-"00000002000000000000003E"
-"00000002000000000000003F"
-"000000020000000000000040"
-"000000020000000000000041"
-"000000020000000000000042"
-"000000020000000000000043"
-"000000020000000000000044"
-"000000020000000000000045"
-"000000020000000000000046"
-"000000020000000000000047"
-"000000020000000000000048"
-"000000020000000000000049"
-"00000002000000000000004A"
-"00000002000000000000004B"
-"00000002000000000000004C"
-"00000002000000000000004D"
-"00000002000000000000004E"
-"00000002000000000000004F"
-"000000020000000000000050"
-"000000020000000000000051"
-"000000020000000000000052"
-"000000020000000000000053"
-"000000020000000000000054"
-"000000020000000000000055"
-"000000020000000000000056"
-"000000020000000000000057"
-"000000020000000000000058"
-"000000020000000000000059"
-"00000002000000000000005A"
-"00000002000000000000005B"
-"00000002000000000000005C"
-"00000002000000000000005D"
-"00000002000000000000005E"
-"00000002000000000000005F"
-"000000020000000000000060"
-"000000020000000000000061"
-"000000020000000000000062"
-"000000020000000000000063"
-"000000020000000000000064"
-"000000020000000000000065"
-"000000020000000000000066"
-"000000020000000000000067"
-"000000020000000000000068"
-"000000020000000000000069"
-"00000002000000000000006A"
-"00000002000000000000006B"
-"00000002000000000000006C"
-"00000002000000000000006D"
-"00000002000000000000006E"
-"00000002000000000000006F"
-"000000020000000000000070"
-"000000020000000000000071"
-"000000020000000000000072"
-"000000020000000000000073"
-"000000020000000000000074"
-"000000020000000000000075"
-"000000020000000000000076"
-"000000020000000000000077"
-"000000020000000000000078"
-"000000020000000000000079"
-"00000002000000000000007A"
-"00000002000000000000007B"
-"00000002000000000000007C"
-"00000002000000000000007D"
-"00000002000000000000007E"
-"00000002000000000000007F"
-"000000020000000000000080"
-"000000020000000000000081"
-"000000020000000000000082"
-"000000020000000000000083"
-"000000020000000000000084"
-"000000020000000000000085"
-"000000020000000000000086"
-"000000020000000000000087"
-"000000020000000000000088"
-"000000020000000000000089"
-"00000002000000000000008A"
-"00000002000000000000008B"
-"00000002000000000000008C"
-"00000002000000000000008D"
-"00000002000000000000008E"
-"00000002000000000000008F"
-"000000020000000000000090"
-"000000020000000000000091"
-"000000020000000000000092"
-"000000020000000000000093"
-"000000020000000000000094"
-"000000020000000000000095"
-"000000020000000000000096"
-"000000020000000000000097"
-"000000020000000000000098"
-"000000020000000000000099"
-"00000002000000000000009A"
-"00000002000000000000009B"
-"00000002000000000000009C"
-"00000002000000000000009D"
-"00000002000000000000009E"
-"00000002000000000000009F"
-"0000000200000000000000A0"
-"0000000200000000000000A1"
-"0000000200000000000000A2"
-"0000000200000000000000A3"
-"0000000200000000000000A4"
-"0000000200000000000000A5"
-"0000000200000000000000A6"
-"0000000200000000000000A7"
-"0000000200000000000000A8"
-"0000000200000000000000A9"
-"0000000200000000000000AA"
-"0000000200000000000000AB"
-"0000000200000000000000AC"
-"0000000200000000000000AD"
-"0000000200000000000000AE"
-"0000000200000000000000AF"
-"0000000200000000000000B0"
-"0000000200000000000000B1"
-"0000000200000000000000B2"
-"0000000200000000000000B3"
-"0000000200000000000000B4"
-"0000000200000000000000B5"
-"0000000200000000000000B6"
-"0000000200000000000000B7"
-"0000000200000000000000B8"
-"0000000200000000000000B9"
-"0000000200000000000000BA"
-"0000000200000000000000BB"
-"0000000200000000000000BC"
-"0000000200000000000000BD"
-"0000000200000000000000BE"
-"0000000200000000000000BF"
-"0000000200000000000000C0"
-"0000000200000000000000C1"
-"0000000200000000000000C2"
-"0000000200000000000000C3"
-"0000000200000000000000C4"
-"0000000200000000000000C5"
-"0000000200000000000000C6"
-"0000000200000000000000C7"
-"0000000200000000000000C8"
-"0000000200000000000000C9"
-"0000000200000000000000CA"
-"0000000200000000000000CB"
-"0000000200000000000000CC"
-"0000000200000000000000CD"
-"0000000200000000000000CE"
-"0000000200000000000000CF"
-"0000000200000000000000D0"
-"0000000200000000000000D1"
-"0000000200000000000000D2"
-"0000000200000000000000D3"
-"0000000200000000000000D4"
-"0000000200000000000000D5"
-"0000000200000000000000D6"
-"0000000200000000000000D7"
-"0000000200000000000000D8"
-"0000000200000000000000D9"
-"0000000200000000000000DA"
-"0000000200000000000000DB"
-"0000000200000000000000DC"
-"0000000200000000000000DD"
-"0000000200000000000000DE"
-"0000000200000000000000DF"
-"0000000200000000000000E0"
-"0000000200000000000000E1"
-"0000000200000000000000E2"
-"0000000200000000000000E3"
-"0000000200000000000000E4"
-"0000000200000000000000E5"
-"0000000200000000000000E6"
-"0000000200000000000000E7"
-"0000000200000000000000E8"
-"0000000200000000000000E9"
-"0000000200000000000000EA"
-"0000000200000000000000EB"
-"0000000200000000000000EC"
-"0000000200000000000000ED"
-"0000000200000000000000EE"
-"0000000200000000000000EF"
-"0000000200000000000000F0"
-"0000000200000000000000F1"
-"0000000200000000000000F2"
-"0000000200000000000000F3"
-"0000000200000000000000F4"
-"0000000200000000000000F5"
-"0000000200000000000000F6"
-"0000000200000000000000F7"
-"0000000200000000000000F8"
-"0000000200000000000000F9"
-"0000000200000000000000FA"
-"0000000200000000000000FB"
-"0000000200000000000000FC"
-"0000000200000000000000FD"
-"0000000200000000000000FE"
-"0000000200000000000000FF"
-"000000020000000000000100"
-"000000020000000000000101"
-"000000020000000000000102"
-"000000020000000000000103"
-"000000020000000000000104"
-"000000020000000000000105"
-"000000020000000000000106"
-"000000020000000000000107"
-"000000020000000000000108"
-"000000020000000000000109"
-"00000002000000000000010A"
-"00000002000000000000010B"
-"00000002000000000000010C"
-"00000002000000000000010D"
-"00000002000000000000010E"
-"00000002000000000000010F"
-"000000020000000000000110"
-"000000020000000000000111"
-"000000020000000000000112"
-"000000020000000000000113"
-"000000020000000000000114"
-"000000020000000000000115"
-"000000020000000000000116"
-"000000020000000000000117"
-"000000020000000000000118"
-"000000020000000000000119"
-"00000002000000000000011A"
-"00000002000000000000011B"
-"00000002000000000000011C"
-"00000002000000000000011D"
-"00000002000000000000011E"
-"00000002000000000000011F"
-"000000020000000000000120"
-"000000020000000000000121"
-"000000020000000000000122"
-"000000020000000000000123"
-"000000020000000000000124"
-"000000020000000000000125"
-"000000020000000000000126"
-"000000020000000000000127"
-"000000020000000000000128"
-"000000020000000000000129"
-"00000002000000000000012A"
-"00000002000000000000012B"
-"00000002000000000000012C"
-"00000002000000000000012D"
-"00000002000000000000012E"
-"00000002000000000000012F"
-"000000020000000000000130"
-"000000020000000000000131"
-"000000020000000000000132"
-"000000020000000000000133"
-"000000020000000000000134"
-"000000020000000000000135"
-"000000020000000000000136"
-"000000020000000000000137"
-"000000020000000000000138"
-"000000020000000000000139"
-"00000002000000000000013A"
-"00000002000000000000013B"
-"00000002000000000000013C"
-"00000002000000000000013D"
-"00000002000000000000013E"
-"00000002000000000000013F"
-"000000020000000000000140"
-"000000020000000000000141"
-"000000020000000000000142"
-"000000020000000000000143"
-"000000020000000000000144"
-"000000020000000000000145"
-"000000020000000000000146"
-"000000020000000000000147"
-"000000020000000000000148"
-"000000020000000000000149"
-"00000002000000000000014A"
-"00000002000000000000014B"
-"00000002000000000000014C"
-"00000002000000000000014D"
-"00000002000000000000014E"
-"00000002000000000000014F"
-"000000020000000000000150"
-"000000020000000000000151"
-"000000020000000000000152"
-"000000020000000000000153"
-"000000020000000000000154"
-"000000020000000000000155"
-"000000020000000000000156"
-"000000020000000000000157"
-"000000020000000000000158"
-"000000020000000000000159"
-"00000002000000000000015A"
-"00000002000000000000015B"
-"00000002000000000000015C"
-"00000002000000000000015D"
-"00000002000000000000015E"
-"00000002000000000000015F"
-"000000020000000000000160"
-"000000020000000000000161"
-"000000020000000000000162"
-"000000020000000000000163"
-"000000020000000000000164"
-"000000020000000000000165"
-"000000020000000000000166"
-"000000020000000000000167"
-"000000020000000000000168"
-"000000020000000000000169"
-"00000002000000000000016A"
-"00000002000000000000016B"
-"00000002000000000000016C"
-"00000002000000000000016D"
-"00000002000000000000016E"
-"00000002000000000000016F"
-"000000020000000000000170"
-"000000020000000000000171"
-"000000020000000000000172"
-"000000020000000000000173"
-"000000020000000000000174"
-"000000020000000000000175"
-"000000020000000000000176"
-"000000020000000000000177"
-"000000020000000000000178"
-"000000020000000000000179"
-"00000002000000000000017A"
-"00000002000000000000017B"
-"00000002000000000000017C"
-"00000002000000000000017D"
-"00000002000000000000017E"
-"00000002000000000000017F"
-"000000020000000000000180"
-"000000020000000000000181"
-"000000020000000000000182"
-"000000020000000000000183"
-"000000020000000000000184"
-"000000020000000000000185"
-"000000020000000000000186"
-"000000020000000000000187"
-"000000020000000000000188"
-"000000020000000000000189"
-"00000002000000000000018A"
-"00000002000000000000018B"
-"00000002000000000000018C"
-"00000002000000000000018D"
-"00000002000000000000018E"
-"00000002000000000000018F"
-"000000020000000000000190"
-"000000020000000000000191"
-"000000020000000000000192"
-"000000020000000000000193"
-"000000020000000000000194"
-"000000020000000000000195"
-"000000020000000000000196"
-"000000020000000000000197"
-"000000020000000000000198"
-"000000020000000000000199"
-"00000002000000000000019A"
-"00000002000000000000019B"
-"00000002000000000000019C"
-"00000002000000000000019D"
-"00000002000000000000019E"
-"00000002000000000000019F"
-"0000000200000000000001A0"
-"0000000200000000000001A1"
-"0000000200000000000001A2"
-"0000000200000000000001A3"
-"0000000200000000000001A4"
-"0000000200000000000001A5"
-"0000000200000000000001A6"
-"0000000200000000000001A7"
-"0000000200000000000001A8"
-"0000000200000000000001A9"
-"0000000200000000000001AA"
-"0000000200000000000001AB"
-"0000000200000000000001AC"
-"0000000200000000000001AD"
-"0000000200000000000001AE"
-"0000000200000000000001AF"
-"0000000200000000000001B0"
-"0000000200000000000001B1"
-"0000000200000000000001B2"
-"0000000200000000000001B3"
-"0000000200000000000001B4"
-"0000000200000000000001B5"
-"0000000200000000000001B6"
-"0000000200000000000001B7"
-"0000000200000000000001B8"
-"0000000200000000000001B9"
-"0000000200000000000001BA"
-"0000000200000000000001BB"
-"0000000200000000000001BC"
-"0000000200000000000001BD"
-"0000000200000000000001BE"
-"0000000200000000000001BF"
-"0000000200000000000001C0"
-"0000000200000000000001C1"
-"0000000200000000000001C2"
-"0000000200000000000001C3"
-"0000000200000000000001C4"
-"0000000200000000000001C5"
-"0000000200000000000001C6"
-"0000000200000000000001C7"
-"0000000200000000000001C8"
-"0000000200000000000001C9"
-"0000000200000000000001CA"
-"0000000200000000000001CB"
-"0000000200000000000001CC"
-"0000000200000000000001CD"
-"0000000200000000000001CE"
-"0000000200000000000001CF"
-"0000000200000000000001D0"
-"0000000200000000000001D1"
-"0000000200000000000001D2"
-"0000000200000000000001D3"
-"0000000200000000000001D4"
-"0000000200000000000001D5"
-"0000000200000000000001D6"
-"0000000200000000000001D7"
-"0000000200000000000001D8"
-"0000000200000000000001D9"
-"0000000200000000000001DA"
-"0000000200000000000001DB"
-"0000000200000000000001DC"
-"0000000200000000000001DD"
-"0000000200000000000001DE"
-"0000000200000000000001DF"
-"0000000200000000000001E0"
-"0000000200000000000001E1"
-"0000000200000000000001E2"
-"0000000200000000000001E3"
-"0000000200000000000001E4"
-"0000000200000000000001E5"
-"0000000200000000000001E6"
-"0000000200000000000001E7"
-"0000000200000000000001E8"
-"0000000200000000000001E9"
-"0000000200000000000001EA"
-"0000000200000000000001EB"
-"0000000200000000000001EC"
-"0000000200000000000001ED"
-"0000000200000000000001EE"
-"0000000200000000000001EF"
-"0000000200000000000001F0"
-"0000000200000000000001F1"
-"0000000200000000000001F2"
-"0000000200000000000001F3"
-"0000000200000000000001F4"
-"0000000200000000000001F5"
-"0000000200000000000001F6"
-"0000000200000000000001F7"
-"0000000200000000000001F8"
-"0000000200000000000001F9"
-"0000000200000000000001FA"
-"0000000200000000000001FB"
-"0000000200000000000001FC"
-"0000000200000000000001FD"
-"0000000200000000000001FE"
-"0000000200000000000001FF"
-"000000020000000000000200"
-"000000020000000000000201"
-"000000020000000000000202"
-"000000020000000000000203"
-"000000020000000000000204"
-"000000020000000000000205"
-"000000020000000000000206"
-"000000020000000000000207"
-"000000020000000000000208"
-"000000020000000000000209"
-"00000002000000000000020A"
-"00000002000000000000020B"
-"00000002000000000000020C"
-"00000002000000000000020D"
-"00000002000000000000020E"
-"00000002000000000000020F"
-"000000020000000000000210"
-"000000020000000000000211"
-"000000020000000000000212"
-"000000020000000000000213"
-"000000020000000000000214"
-"000000020000000000000215"
-"000000020000000000000216"
-"000000020000000000000217"
-"000000020000000000000218"
-"000000020000000000000219"
-"00000002000000000000021A"
-"00000002000000000000021B"
-"00000002000000000000021C"
-"00000002000000000000021D"
-"00000002000000000000021E"
-"00000002000000000000021F"
-"000000020000000000000220"
-"000000020000000000000221"
-"000000020000000000000222"
-"000000020000000000000223"
-"000000020000000000000224"
-"000000020000000000000225"
-"000000020000000000000226"
-"000000020000000000000227"
-"000000020000000000000228"
-"000000020000000000000229"
-"00000002000000000000022A"
-"00000002000000000000022B"
-"00000002000000000000022C"
-"00000002000000000000022D"
-"00000002000000000000022E"
-"00000002000000000000022F"
-"000000020000000000000230"
-"000000020000000000000231"
-"000000020000000000000232"
-"000000020000000000000233"
-"000000020000000000000234"
-"000000020000000000000235"
-"000000020000000000000236"
-"000000020000000000000237"
-"000000020000000000000238"
-"000000020000000000000239"
-"00000002000000000000023A"
-"00000002000000000000023B"
-"00000002000000000000023C"
-"00000002000000000000023D"
-"00000002000000000000023E"
-"00000002000000000000023F"
-"000000020000000000000240"
-"000000020000000000000241"
-"000000020000000000000242"
-"000000020000000000000243"
-"000000020000000000000244"
-"000000020000000000000245"
-"000000020000000000000246"
-"000000020000000000000247"
-"000000020000000000000248"
-"000000020000000000000249"
-"00000002000000000000024A"
-"00000002000000000000024B"
-"00000002000000000000024C"
-"00000002000000000000024D"
-"00000002000000000000024E"
-"00000002000000000000024F"
-"000000020000000000000250"
-"000000020000000000000251"
-"000000020000000000000252"
-"000000020000000000000253"
-"000000020000000000000254"
-"000000020000000000000255"
-"000000020000000000000256"
-"000000020000000000000257"
-"000000020000000000000258"
-"000000020000000000000259"
-"00000002000000000000025A"
-"00000002000000000000025B"
-"00000002000000000000025C"
-"00000002000000000000025D"
-"00000002000000000000025E"
-"00000002000000000000025F"
-"000000020000000000000260"
-"000000020000000000000261"
-"000000020000000000000262"
-"000000020000000000000263"
-"000000020000000000000264"
-"000000020000000000000265"
-"000000020000000000000266"
-"000000020000000000000267"
-"000000020000000000000268"
-"000000020000000000000269"
-"00000002000000000000026A"
-"00000002000000000000026B"
-"00000002000000000000026C"
-"00000002000000000000026D"
-"00000002000000000000026E"
-"00000002000000000000026F"
-"000000020000000000000270"
-"000000020000000000000271"
-"000000020000000000000272"
-"000000020000000000000273"
-"000000020000000000000274"
-"000000020000000000000275"
-"000000020000000000000276"
-"000000020000000000000277"
-"000000020000000000000278"
-"000000020000000000000279"
-"00000002000000000000027A"
-"00000002000000000000027B"
-"00000002000000000000027C"
-"00000002000000000000027D"
-"00000002000000000000027E"
-"00000002000000000000027F"
-"000000020000000000000280"
-"000000020000000000000281"
-"000000020000000000000282"
-"000000020000000000000283"
-"000000020000000000000284"
-"000000020000000000000285"
-"000000020000000000000286"
-"000000020000000000000287"
-"000000020000000000000288"
-"000000020000000000000289"
-"00000002000000000000028A"
-"00000002000000000000028B"
-"00000002000000000000028C"
-"00000002000000000000028D"
-"00000002000000000000028E"
-"00000002000000000000028F"
-"000000020000000000000290"
-"000000020000000000000291"
-"000000020000000000000292"
-"000000020000000000000293"
-"000000020000000000000294"
-"000000020000000000000295"
-"000000020000000000000296"
-"000000020000000000000297"
-"000000020000000000000298"
-"000000020000000000000299"
-"00000002000000000000029A"
-"00000002000000000000029B"
-"00000002000000000000029C"
-"00000002000000000000029D"
-"00000002000000000000029E"
-"00000002000000000000029F"
-"0000000200000000000002A0"
-"0000000200000000000002A1"
-"0000000200000000000002A2"
-"0000000200000000000002A3"
-"0000000200000000000002A4"
-"0000000200000000000002A5"
-"0000000200000000000002A6"
-"0000000200000000000002A7"
-"0000000200000000000002A8"
-"0000000200000000000002A9"
-"0000000200000000000002AA"
-"0000000200000000000002AB"
-"0000000200000000000002AC"
-"0000000200000000000002AD"
-"0000000200000000000002AE"
-"0000000200000000000002AF"
-"0000000200000000000002B0"
-"0000000200000000000002B1"
-"0000000200000000000002B2"
-"0000000200000000000002B3"
-"0000000200000000000002B4"
-"0000000200000000000002B5"
-"0000000200000000000002B6"
-"0000000200000000000002B7"
-"0000000200000000000002B8"
-"0000000200000000000002B9"
-"0000000200000000000002BA"
-"0000000200000000000002BB"
-"0000000200000000000002BC"
-"0000000200000000000002BD"
-"0000000200000000000002BE"
-"0000000200000000000002BF"
-"0000000200000000000002C0"
-"0000000200000000000002C1"
-"0000000200000000000002C2"
-"0000000200000000000002C3"
-"0000000200000000000002C4"
-"0000000200000000000002C5"
-"0000000200000000000002C6"
-"0000000200000000000002C7"
-"0000000200000000000002C8"
-"0000000200000000000002C9"
-"0000000200000000000002CA"
-"0000000200000000000002CB"
-"0000000200000000000002CC"
-"0000000200000000000002CD"
-"0000000200000000000002CE"
-"0000000200000000000002CF"
-"0000000200000000000002D0"
-"0000000200000000000002D1"
-"0000000200000000000002D2"
-"0000000200000000000002D3"
-"0000000200000000000002D4"
-"0000000200000000000002D5"
-"0000000200000000000002D6"
-"0000000200000000000002D7"
-"0000000200000000000002D8"
-"0000000200000000000002D9"
-"0000000200000000000002DA"
-"0000000200000000000002DB"
-"0000000200000000000002DC"
-"0000000200000000000002DD"
-"0000000200000000000002DE"
-"0000000200000000000002DF"
-"0000000200000000000002E0"
-"0000000200000000000002E1"
-"0000000200000000000002E2"
-"0000000200000000000002E3"
-"0000000200000000000002E4"
-"0000000200000000000002E5"
-"0000000200000000000002E6"
-"0000000200000000000002E7"
-"0000000200000000000002E8"
-"0000000200000000000002E9"
-"0000000200000000000002EA"
-"0000000200000000000002EB"
-"0000000200000000000002EC"
-"0000000200000000000002ED"
-"0000000200000000000002EE"
-"0000000200000000000002EF"
-"0000000200000000000002F0"
-"0000000200000000000002F1"
-"0000000200000000000002F2"
-"0000000200000000000002F3"
-"0000000200000000000002F4"
-"0000000200000000000002F5"
-"0000000200000000000002F6"
-"0000000200000000000002F7"
-"0000000200000000000002F8"
-"0000000200000000000002F9"
-"0000000200000000000002FA"
-"0000000200000000000002FB"
-"0000000200000000000002FC"
-"0000000200000000000002FD"
-"0000000200000000000002FE"
-"0000000200000000000002FF"
-"000000020000000000000300"
-"000000020000000000000301"
-"000000020000000000000302"
-"000000020000000000000303"
-"000000020000000000000304"
-"000000020000000000000305"
-"000000020000000000000306"
-"000000020000000000000307"
-"000000020000000000000308"
-"000000020000000000000309"
-"00000002000000000000030A"
-"00000002000000000000030B"
-"00000002000000000000030C"
-"00000002000000000000030D"
-"00000002000000000000030E"
-"00000002000000000000030F"
-"000000020000000000000310"
-"000000020000000000000311"
-"000000020000000000000312"
-"000000020000000000000313"
-"000000020000000000000314"
-"000000020000000000000315"
-"000000020000000000000316"
-"000000020000000000000317"
-"000000020000000000000318"
-"000000020000000000000319"
-"00000002000000000000031A"
-"00000002000000000000031B"
-"00000002000000000000031C"
-"00000002000000000000031D"
-"00000002000000000000031E"
-"00000002000000000000031F"
-"000000020000000000000320"
-"000000020000000000000321"
-"000000020000000000000322"
-"000000020000000000000323"
-"000000020000000000000324"
-"000000020000000000000325"
-"000000020000000000000326"
-"000000020000000000000327"
-"000000020000000000000328"
-"000000020000000000000329"
-"00000002000000000000032A"
-"00000002000000000000032B"
-"00000002000000000000032C"
-"00000002000000000000032D"
-"00000002000000000000032E"
-"00000002000000000000032F"
-"000000020000000000000330"
-"000000020000000000000331"
-"000000020000000000000332"
-"000000020000000000000333"
-"000000020000000000000334"
-"000000020000000000000335"
-"000000020000000000000336"
-"000000020000000000000337"
-"000000020000000000000338"
-"000000020000000000000339"
-"00000002000000000000033A"
-"00000002000000000000033B"
-"00000002000000000000033C"
-"00000002000000000000033D"
-"00000002000000000000033E"
-"00000002000000000000033F"
-"000000020000000000000340"
-"000000020000000000000341"
-"000000020000000000000342"
-"000000020000000000000343"
-"000000020000000000000344"
-"000000020000000000000345"
-"000000020000000000000346"
-"000000020000000000000347"
-"000000020000000000000348"
-"000000020000000000000349"
-"00000002000000000000034A"
-"00000002000000000000034B"
-"00000002000000000000034C"
-"00000002000000000000034D"
-"00000002000000000000034E"
-"00000002000000000000034F"
-"000000020000000000000350"
-"000000020000000000000351"
-"000000020000000000000352"
-"000000020000000000000353"
-"000000020000000000000354"
-"000000020000000000000355"
-"000000020000000000000356"
-"000000020000000000000357"
-"000000020000000000000358"
-"000000020000000000000359"
-"00000002000000000000035A"
-"00000002000000000000035B"
-"00000002000000000000035C"
-"00000002000000000000035D"
-"00000002000000000000035E"
-"00000002000000000000035F"
-"000000020000000000000360"
-"000000020000000000000361"
-"000000020000000000000362"
-"000000020000000000000363"
-"000000020000000000000364"
-"000000020000000000000365"
-"000000020000000000000366"
-"000000020000000000000367"
-"000000020000000000000368"
-"000000020000000000000369"
-"00000002000000000000036A"
-"00000002000000000000036B"
-"00000002000000000000036C"
-"00000002000000000000036D"
-"00000002000000000000036E"
-"00000002000000000000036F"
-"000000020000000000000370"
-"000000020000000000000371"
-"000000020000000000000372"
-"000000020000000000000373"
-"000000020000000000000374"
-"000000020000000000000375"
-"000000020000000000000376"
-"000000020000000000000377"
-"000000020000000000000378"
-"000000020000000000000379"
-"00000002000000000000037A"
-"00000002000000000000037B"
-"00000002000000000000037C"
-"00000002000000000000037D"
-"00000002000000000000037E"
-"00000002000000000000037F"
-"000000020000000000000380"
-"000000020000000000000381"
-"000000020000000000000382"
-"000000020000000000000383"
-"000000020000000000000384"
-"000000020000000000000385"
-"000000020000000000000386"
-"000000020000000000000387"
-"000000020000000000000388"
-"000000020000000000000389"
-"00000002000000000000038A"
-"00000002000000000000038B"
-"00000002000000000000038C"
-"00000002000000000000038D"
-"00000002000000000000038E"
-"00000002000000000000038F"
-"000000020000000000000390"
-"000000020000000000000391"
-"000000020000000000000392"
-"000000020000000000000393"
-"000000020000000000000394"
-"000000020000000000000395"
-"000000020000000000000396"
-"000000020000000000000397"
-"000000020000000000000398"
-"000000020000000000000399"
-"00000002000000000000039A"
-"00000002000000000000039B"
-"00000002000000000000039C"
-"00000002000000000000039D"
-"00000002000000000000039E"
-"00000002000000000000039F"
-"0000000200000000000003A0"
-"0000000200000000000003A1"
-"0000000200000000000003A2"
-"0000000200000000000003A3"
-"0000000200000000000003A4"
-"0000000200000000000003A5"
-"0000000200000000000003A6"
-"0000000200000000000003A7"
-"0000000200000000000003A8"
-"0000000200000000000003A9"
-"0000000200000000000003AA"
-"0000000200000000000003AB"
-"0000000200000000000003AC"
-"0000000200000000000003AD"
-"0000000200000000000003AE"
-"0000000200000000000003AF"
-"0000000200000000000003B0"
-"0000000200000000000003B1"
-"0000000200000000000003B2"
-"0000000200000000000003B3"
-"0000000200000000000003B4"
-"0000000200000000000003B5"
-"0000000200000000000003B6"
-"0000000200000000000003B7"
-"0000000200000000000003B8"
-"0000000200000000000003B9"
-"0000000200000000000003BA"
-"0000000200000000000003BB"
-"0000000200000000000003BC"
-"0000000200000000000003BD"
-"0000000200000000000003BE"
-"0000000200000000000003BF"
-"0000000200000000000003C0"
-"0000000200000000000003C1"
-"0000000200000000000003C2"
-"0000000200000000000003C3"
-"0000000200000000000003C4"
-"0000000200000000000003C5"
-"0000000200000000000003C6"
-"0000000200000000000003C7"
-"0000000200000000000003C8"
-"0000000200000000000003C9"
-"0000000200000000000003CA"
-"0000000200000000000003CB"
-"0000000200000000000003CC"
-"0000000200000000000003CD"
-"0000000200000000000003CE"
-"0000000200000000000003CF"
-"0000000200000000000003D0"
-"0000000200000000000003D1"
-"0000000200000000000003D2"
-"0000000200000000000003D3"
-"0000000200000000000003D4"
-"0000000200000000000003D5"
-"0000000200000000000003D6"
-"0000000200000000000003D7"
-"0000000200000000000003D8"
-"0000000200000000000003D9"
-"0000000200000000000003DA"
-"0000000200000000000003DB"
-"0000000200000000000003DC"
-"0000000200000000000003DD"
-"0000000200000000000003DE"
-"0000000200000000000003DF"
-"0000000200000000000003E0"
-"0000000200000000000003E1"
-"0000000200000000000003E2"
-"0000000200000000000003E3"
-"0000000200000000000003E4"
-"0000000200000000000003E5"
-"0000000200000000000003E6"
-"0000000200000000000003E7"
-"0000000200000000000003E8"
-"0000000200000000000003E9"
-"0000000200000000000003EA"
-"0000000200000000000003EB"
-"0000000200000000000003EC"
-"0000000200000000000003ED"
-"0000000200000000000003EE"
-"0000000200000000000003EF"
-"0000000200000000000003F0"
-"0000000200000000000003F1"
-"0000000200000000000003F2"
-"0000000200000000000003F3"
-"0000000200000000000003F4"
-"0000000200000000000003F5"
-"0000000200000000000003F6"
-"0000000200000000000003F7"
-"0000000200000000000003F8"
-"0000000200000000000003F9"
-"0000000200000000000003FA"
-"0000000200000000000003FB"
-"0000000200000000000003FC"
-"0000000200000000000003FD"
-"0000000200000000000003FE"
-"0000000200000000000003FF"
-"000000020000000000000400"
-"000000020000000000000401"
-"000000020000000000000402"
-"000000020000000000000403"
-"000000020000000000000404"
-"000000020000000000000405"
-"000000020000000000000406"
-"000000020000000000000407"
-"000000020000000000000408"
-"000000020000000000000409"
-"00000002000000000000040A"
-"00000002000000000000040B"
-"00000002000000000000040C"
-"00000002000000000000040D"
-"00000002000000000000040E"
-"00000002000000000000040F"
-"000000020000000000000410"
-"000000020000000000000411"
-"000000020000000000000412"
-"000000020000000000000413"
-"000000020000000000000414"
-"000000020000000000000415"
-"000000020000000000000416"
-"000000020000000000000417"
-"000000020000000000000418"
-"000000020000000000000419"
-"00000002000000000000041A"
-"00000002000000000000041B"
-"00000002000000000000041C"
-"00000002000000000000041D"
-"00000002000000000000041E"
-"00000002000000000000041F"
-"000000020000000000000420"
-"000000020000000000000421"
-"000000020000000000000422"
-"000000020000000000000423"
-"000000020000000000000424"
-"000000020000000000000425"
-"000000020000000000000426"
-"000000020000000000000427"
-"000000020000000000000428"
-"000000020000000000000429"
-"00000002000000000000042A"
-"00000002000000000000042B"
-"00000002000000000000042C"
-"00000002000000000000042D"
-"00000002000000000000042E"
-"00000002000000000000042F"
-"000000020000000000000430"
-"000000020000000000000431"
-"000000020000000000000432"
-"000000020000000000000433"
-"000000020000000000000434"
-"000000020000000000000435"
-"000000020000000000000436"
-"000000020000000000000437"
-"000000020000000000000438"
-"000000020000000000000439"
-"00000002000000000000043A"
-"00000002000000000000043B"
-"00000002000000000000043C"
-"00000002000000000000043D"
-"00000002000000000000043E"
-"00000002000000000000043F"
-"000000020000000000000440"
-"000000020000000000000441"
-"000000020000000000000442"
-"000000020000000000000443"
-"000000020000000000000444"
-"000000020000000000000445"
-"000000020000000000000446"
-"000000020000000000000447"
-"000000020000000000000448"
-"000000020000000000000449"
-"00000002000000000000044A"
-"00000002000000000000044B"
-"00000002000000000000044C"
-"00000002000000000000044D"
-"00000002000000000000044E"
-"00000002000000000000044F"
-"000000020000000000000450"
-"000000020000000000000451"
-"000000020000000000000452"
-"000000020000000000000453"
-"000000020000000000000454"
-"000000020000000000000455"
-"000000020000000000000456"
-"000000020000000000000457"
-"000000020000000000000458"
-"000000020000000000000459"
-"00000002000000000000045A"
-"00000002000000000000045B"
-"00000002000000000000045C"
-"00000002000000000000045D"
-"00000002000000000000045E"
-"00000002000000000000045F"
-"000000020000000000000460"
-"000000020000000000000461"
-"000000020000000000000462"
-"000000020000000000000463"
-"000000020000000000000464"
-"000000020000000000000465"
-"000000020000000000000466"
-"000000020000000000000467"
-"000000020000000000000468"
-"000000020000000000000469"
-"00000002000000000000046A"
-"00000002000000000000046B"
-"00000002000000000000046C"
-"00000002000000000000046D"
-"00000002000000000000046E"
-"00000002000000000000046F"
-"000000020000000000000470"
-"000000020000000000000471"
-"000000020000000000000472"
-"000000020000000000000473"
-"000000020000000000000474"
-"000000020000000000000475"
-"000000020000000000000476"
-"000000020000000000000477"
-"000000020000000000000478"
-"000000020000000000000479"
-"00000002000000000000047A"
-"00000002000000000000047B"
-"00000002000000000000047C"
-"00000002000000000000047D"
-"00000002000000000000047E"
-"00000002000000000000047F"
-"000000020000000000000480"
-"000000020000000000000481"
-"000000020000000000000482"
-"000000020000000000000483"
-"000000020000000000000484"
-"000000020000000000000485"
-"000000020000000000000486"
-"000000020000000000000487"
-"000000020000000000000488"
-"000000020000000000000489"
-"00000002000000000000048A"
-"00000002000000000000048B"
-"00000002000000000000048C"
-"00000002000000000000048D"
-"00000002000000000000048E"
-"00000002000000000000048F"
-"000000020000000000000490"
-"000000020000000000000491"
-"000000020000000000000492"
-"000000020000000000000493"
-"000000020000000000000494"
-"000000020000000000000495"
-"000000020000000000000496"
-"000000020000000000000497"
-"000000020000000000000498"
-"000000020000000000000499"
-"00000002000000000000049A"
-"00000002000000000000049B"
-"00000002000000000000049C"
-"00000002000000000000049D"
-"00000002000000000000049E"
-"00000002000000000000049F"
-"0000000200000000000004A0"
-"0000000200000000000004A1"
-"0000000200000000000004A2"
-"0000000200000000000004A3"
-"0000000200000000000004A4"
-"0000000200000000000004A5"
-"0000000200000000000004A6"
-"0000000200000000000004A7"
-"0000000200000000000004A8"
-"0000000200000000000004A9"
-"0000000200000000000004AA"
-"0000000200000000000004AB"
-"0000000200000000000004AC"
-"0000000200000000000004AD"
-"0000000200000000000004AE"
-"0000000200000000000004AF"
-"0000000200000000000004B0"
-"0000000200000000000004B1"
-"0000000200000000000004B2"
-"0000000200000000000004B3"
-"0000000200000000000004B4"
-"0000000200000000000004B5"
-"0000000200000000000004B6"
-"0000000200000000000004B7"
-"0000000200000000000004B8"
-"0000000200000000000004B9"
-"0000000200000000000004BA"
-"0000000200000000000004BB"
-"0000000200000000000004BC"
-"0000000200000000000004BD"
-"0000000200000000000004BE"
-"0000000200000000000004BF"
-"0000000200000000000004C0"
-"0000000200000000000004C1"
-"0000000200000000000004C2"
-"0000000200000000000004C3"
-"0000000200000000000004C4"
-"0000000200000000000004C5"
-"0000000200000000000004C6"
-"0000000200000000000004C7"
-"0000000200000000000004C8"
-"0000000200000000000004C9"
-"0000000200000000000004CA"
-"0000000200000000000004CB"
-"0000000200000000000004CC"
-"0000000200000000000004CD"
-"0000000200000000000004CE"
-"0000000200000000000004CF"
-"0000000200000000000004D0"
-"0000000200000000000004D1"
-"0000000200000000000004D2"
-"0000000200000000000004D3"
-"0000000200000000000004D4"
-"0000000200000000000004D5"
-"0000000200000000000004D6"
-"0000000200000000000004D7"
-"0000000200000000000004D8"
-"0000000200000000000004D9"
-"0000000200000000000004DA"
-"0000000200000000000004DB"
-"0000000200000000000004DC"
-"0000000200000000000004DD"
-"0000000200000000000004DE"
-"0000000200000000000004DF"
-"0000000200000000000004E0"
-"0000000200000000000004E1"
-"0000000200000000000004E2"
-"0000000200000000000004E3"
-"0000000200000000000004E4"
-"0000000200000000000004E5"
-"0000000200000000000004E6"
-"0000000200000000000004E7"
-"0000000200000000000004E8"
-"0000000200000000000004E9"
-"0000000200000000000004EA"
-"0000000200000000000004EB"
-"0000000200000000000004EC"
-"0000000200000000000004ED"
-"0000000200000000000004EE"
-"0000000200000000000004EF"
-"0000000200000000000004F0"
-"0000000200000000000004F1"
-"0000000200000000000004F2"
-"0000000200000000000004F3"
-"0000000200000000000004F4"
-"0000000200000000000004F5"
-"0000000200000000000004F6"
-"0000000200000000000004F7"
-"0000000200000000000004F8"
-"0000000200000000000004F9"
-"0000000200000000000004FA"
-"0000000200000000000004FB"
-"0000000200000000000004FC"
-"0000000200000000000004FD"
-"0000000200000000000004FE"
-"0000000200000000000004FF"
-"000000020000000000000500"
-"000000020000000000000501"
-"000000020000000000000502"
-"000000020000000000000503"
-"000000020000000000000504"
-"000000020000000000000505"
-"000000020000000000000506"
-"000000020000000000000507"
-"000000020000000000000508"
-"000000020000000000000509"
-"00000002000000000000050A"
-"00000002000000000000050B"
-"00000002000000000000050C"
-"00000002000000000000050D"
-"00000002000000000000050E"
-"00000002000000000000050F"
-"000000020000000000000510"
-"000000020000000000000511"
-"000000020000000000000512"
-"000000020000000000000513"
-"000000020000000000000514"
-"000000020000000000000515"
-"000000020000000000000516"
-"000000020000000000000517"
-"000000020000000000000518"
-"000000020000000000000519"
-"00000002000000000000051A"
-"00000002000000000000051B"
-"00000002000000000000051C"
-"00000002000000000000051D"
-"00000002000000000000051E"
-"00000002000000000000051F"
-"000000020000000000000520"
-"000000020000000000000521"
-"000000020000000000000522"
-"000000020000000000000523"
-"000000020000000000000524"
-"000000020000000000000525"
-"000000020000000000000526"
-"000000020000000000000527"
-"000000020000000000000528"
-"000000020000000000000529"
-"00000002000000000000052A"
-"00000002000000000000052B"
-"00000002000000000000052C"
-"00000002000000000000052D"
-"00000002000000000000052E"
-"00000002000000000000052F"
-"000000020000000000000530"
-"000000020000000000000531"
-"000000020000000000000532"
-"000000020000000000000533"
-"000000020000000000000534"
-"000000020000000000000535"
-"000000020000000000000536"
-"000000020000000000000537"
-"000000020000000000000538"
-"000000020000000000000539"
-"00000002000000000000053A"
-"00000002000000000000053B"
-"00000002000000000000053C"
-"00000002000000000000053D"
-"00000002000000000000053E"
-"00000002000000000000053F"
-"000000020000000000000540"
-"000000020000000000000541"
-"000000020000000000000542"
-"000000020000000000000543"
-"000000020000000000000544"
-"000000020000000000000545"
-"000000020000000000000546"
-"000000020000000000000547"
-"000000020000000000000548"
-"000000020000000000000549"
-"00000002000000000000054A"
-"00000002000000000000054B"
-"00000002000000000000054C"
-"00000002000000000000054D"
-"00000002000000000000054E"
-"00000002000000000000054F"
-"000000020000000000000550"
-"000000020000000000000551"
-"000000020000000000000552"
-"000000020000000000000553"
-"000000020000000000000554"
-"000000020000000000000555"
-"000000020000000000000556"
-"000000020000000000000557"
-"000000020000000000000558"
-"000000020000000000000559"
-"00000002000000000000055A"
-"00000002000000000000055B"
-"00000002000000000000055C"
-"00000002000000000000055D"
-"00000002000000000000055E"
-"00000002000000000000055F"
-"000000020000000000000560"
-"000000020000000000000561"
-"000000020000000000000562"
-"000000020000000000000563"
-"000000020000000000000564"
-"000000020000000000000565"
-"000000020000000000000566"
-"000000020000000000000567"
-"000000020000000000000568"
-"000000020000000000000569"
-"00000002000000000000056A"
-"00000002000000000000056B"
-"00000002000000000000056C"
-"00000002000000000000056D"
-"00000002000000000000056E"
-"00000002000000000000056F"
-"000000020000000000000570"
-"000000020000000000000571"
-"000000020000000000000572"
-"000000020000000000000573"
-"000000020000000000000574"
-"000000020000000000000575"
-"000000020000000000000576"
-"000000020000000000000577"
-"000000020000000000000578"
-"000000020000000000000579"
-"00000002000000000000057A"
-"00000002000000000000057B"
-"00000002000000000000057C"
-"00000002000000000000057D"
-"00000002000000000000057E"
-"00000002000000000000057F"
-"000000020000000000000580"
-"000000020000000000000581"
-"000000020000000000000582"
-"000000020000000000000583"
-"000000020000000000000584"
-"000000020000000000000585"
-"000000020000000000000586"
-"000000020000000000000587"
-"000000020000000000000588"
-"000000020000000000000589"
-"00000002000000000000058A"
-"00000002000000000000058B"
-"00000002000000000000058C"
-"00000002000000000000058D"
-"00000002000000000000058E"
-"00000002000000000000058F"
-"000000020000000000000590"
-"000000020000000000000591"
-"000000020000000000000592"
-"000000020000000000000593"
-"000000020000000000000594"
-"000000020000000000000595"
-"000000020000000000000596"
-"000000020000000000000597"
-"000000020000000000000598"
-"000000020000000000000599"
-"00000002000000000000059A"
-"00000002000000000000059B"
-"00000002000000000000059C"
-"00000002000000000000059D"
-"00000002000000000000059E"
-"00000002000000000000059F"
-"0000000200000000000005A0"
-"0000000200000000000005A1"
-"0000000200000000000005A2"
-"0000000200000000000005A3"
-"0000000200000000000005A4"
-"0000000200000000000005A5"
-"0000000200000000000005A6"
-"0000000200000000000005A7"
-"0000000200000000000005A8"
-"0000000200000000000005A9"
-"0000000200000000000005AA"
-"0000000200000000000005AB"
-"0000000200000000000005AC"
-"0000000200000000000005AD"
-"0000000200000000000005AE"
-"0000000200000000000005AF"
-"0000000200000000000005B0"
-"0000000200000000000005B1"
-"0000000200000000000005B2"
-"0000000200000000000005B3"
-"0000000200000000000005B4"
-"0000000200000000000005B5"
-"0000000200000000000005B6"
-"0000000200000000000005B7"
-"0000000200000000000005B8"
-"0000000200000000000005B9"
-"0000000200000000000005BA"
-"000000030000000000000001"
-"000000030000000000000002"
-"000000030000000000000003"
-"000000030000000000000004"
-"000000030000000000000005"
-"000000030000000000000006"
-"000000030000000000000007"
-"000000030000000000000008"
-"000000030000000000000009"
-"00000003000000000000000A"
-"00000003000000000000000B"
-"00000003000000000000000C"
-"00000003000000000000000D"
-"00000003000000000000000E"
-"00000003000000000000000F"
-"000000030000000000000010"
-"000000030000000000000011"
-"000000030000000000000012"
-"000000030000000000000013"
-"000000030000000000000014"
-"000000030000000000000015"
-"000000030000000000000016"
-"000000030000000000000017"
-"000000030000000000000018"
-"000000030000000000000019"
-"00000003000000000000001A"
-"00000003000000000000001B"
-"00000003000000000000001C"
-"00000003000000000000001D"
-"00000003000000000000001E"
-"00000003000000000000001F"
-"000000030000000000000020"
-"000000030000000000000021"
-"000000030000000000000022"
-"000000030000000000000023"
-"000000030000000000000024"
-"000000030000000000000025"
-"000000030000000000000026"
-"000000030000000000000027"
-"000000030000000000000028"
-"000000030000000000000029"
-"00000003000000000000002A"
-"00000003000000000000002B"
-"00000003000000000000002C"
-"00000003000000000000002D"
-"00000003000000000000002E"
-"00000003000000000000002F"
-"000000030000000000000030"
-"000000030000000000000031"
-"000000030000000000000032"
-"000000030000000000000033"
-"000000030000000000000034"
-"000000030000000000000035"
-"000000030000000000000036"
-"000000030000000000000037"
-"000000030000000000000038"
-"000000030000000000000039"
-"00000003000000000000003A"
-"00000003000000000000003B"
-"00000003000000000000003C"
-"00000003000000000000003D"
-"00000003000000000000003E"
-"00000003000000000000003F"
-"000000030000000000000040"
-"000000030000000000000041"
-"000000030000000000000042"
-"000000030000000000000043"
-"000000030000000000000044"
-"000000030000000000000045"
-"000000030000000000000046"
-"000000030000000000000047"
-"000000030000000000000048"
-"000000030000000000000049"
-"00000003000000000000004A"
-"00000003000000000000004B"
-"00000003000000000000004C"
-"00000003000000000000004D"
-"00000003000000000000004E"
-"00000003000000000000004F"
-"000000030000000000000050"
-"000000030000000000000051"
-"000000030000000000000052"
-"000000030000000000000053"
-"000000030000000000000054"
-"000000030000000000000055"
-"000000030000000000000056"
-"000000030000000000000057"
-"000000030000000000000058"
-"000000030000000000000059"
-"00000003000000000000005A"
-"00000003000000000000005B"
-"00000003000000000000005C"
-"00000003000000000000005D"
-"00000003000000000000005E"
-"00000003000000000000005F"
-"000000030000000000000060"
-"000000030000000000000061"
-"000000030000000000000062"
-"000000030000000000000063"
-"000000030000000000000064"
-"000000030000000000000065"
-"000000030000000000000066"
-"000000030000000000000067"
-"000000030000000000000068"
-"000000030000000000000069"
-"00000003000000000000006A"
-"00000003000000000000006B"
-"00000003000000000000006C"
-"00000003000000000000006D"
-"00000003000000000000006E"
-"00000003000000000000006F"
-"000000030000000000000070"
-"000000030000000000000071"
-"000000030000000000000072"
-"000000030000000000000073"
-"000000030000000000000074"
-"000000030000000000000075"
-"000000030000000000000076"
-"000000030000000000000077"
-"000000030000000000000078"
-"000000030000000000000079"
-"00000003000000000000007A"
-"00000003000000000000007B"
-"00000003000000000000007C"
-"00000003000000000000007D"
-"00000003000000000000007E"
-"00000003000000000000007F"
-"000000030000000000000080"
-"000000030000000000000081"
-"000000030000000000000082"
-"000000030000000000000083"
-"000000030000000000000084"
-"000000030000000000000085"
-"000000030000000000000086"
-"000000030000000000000087"
-"000000030000000000000088"
-"000000030000000000000089"
-"00000003000000000000008A"
-"00000003000000000000008B"
-"00000003000000000000008C"
-"00000003000000000000008D"
-"00000003000000000000008E"
-"00000003000000000000008F"
-"000000030000000000000090"
-"000000030000000000000091"
-"000000030000000000000092"
-"000000030000000000000093"
-"000000030000000000000094"
-"000000030000000000000095"
-"000000030000000000000096"
-"000000030000000000000097"
-"000000030000000000000098"
-"000000030000000000000099"
-"00000003000000000000009A"
-"00000003000000000000009B"
-"00000003000000000000009C"
-"00000003000000000000009D"
-"00000003000000000000009E"
-"00000003000000000000009F"
-"0000000300000000000000A0"
-"0000000300000000000000A1"
-"0000000300000000000000A2"
-"0000000300000000000000A3"
-"0000000300000000000000A4"
-"0000000300000000000000A5"
-"0000000300000000000000A6"
-"0000000300000000000000A7"
-"0000000300000000000000A8"
-"0000000300000000000000A9"
-"0000000300000000000000AA"
-"0000000300000000000000AB"
-"0000000300000000000000AC"
-"0000000300000000000000AD"
-"0000000300000000000000AE"
-"0000000300000000000000AF"
-"0000000300000000000000B0"
-"0000000300000000000000B1"
-"0000000300000000000000B2"
-"0000000300000000000000B3"
-"0000000300000000000000B4"
-"0000000300000000000000B5"
-"0000000300000000000000B6"
-"0000000300000000000000B7"
-"0000000300000000000000B8"
-"0000000300000000000000B9"
-"0000000300000000000000BA"
-"0000000300000000000000BB"
-"0000000300000000000000BC"
-"0000000300000000000000BD"
-"0000000300000000000000BE"
-"0000000300000000000000BF"
-"0000000300000000000000C0"
-"0000000300000000000000C1"
-"0000000300000000000000C2"
-"0000000300000000000000C3"
-"0000000300000000000000C4"
-"0000000300000000000000C5"
-"0000000300000000000000C6"
-"0000000300000000000000C7"
-"0000000300000000000000C8"
-"0000000300000000000000C9"
-"0000000300000000000000CA"
-"0000000300000000000000CB"
-"0000000300000000000000CC"
-"0000000300000000000000CD"
-"0000000300000000000000CE"
-"0000000300000000000000CF"
-"0000000300000000000000D0"
-"0000000300000000000000D1"
-"0000000300000000000000D2"
-"0000000300000000000000D3"
-"0000000300000000000000D4"
-"0000000300000000000000D5"
-"0000000300000000000000D6"
-"0000000300000000000000D7"
-"0000000300000000000000D8"
-"0000000300000000000000D9"
-"0000000300000000000000DA"
-"0000000300000000000000DB"
-"0000000300000000000000DC"
-"0000000300000000000000DD"
-"0000000300000000000000DE"
-"0000000300000000000000DF"
-"0000000300000000000000E0"
-"0000000300000000000000E1"
-"0000000300000000000000E2"
-"0000000300000000000000E3"
-"0000000300000000000000E4"
-"0000000300000000000000E5"
-"0000000300000000000000E6"
-"0000000300000000000000E7"
-"0000000300000000000000E8"
-"0000000300000000000000E9"
-"0000000300000000000000EA"
-"0000000300000000000000EB"
-"0000000300000000000000EC"
-"0000000300000000000000ED"
-"0000000300000000000000EE"
-"0000000300000000000000EF"
-"0000000300000000000000F0"
-"0000000300000000000000F1"
-"0000000300000000000000F2"
-"0000000300000000000000F3"
-"0000000300000000000000F4"
-"0000000300000000000000F5"
-"0000000300000000000000F6"
-"0000000300000000000000F7"
-"0000000300000000000000F8"
-"0000000300000000000000F9"
-"0000000300000000000000FA"
-"0000000300000000000000FB"
-"0000000300000000000000FC"
-"0000000300000000000000FD"
-"0000000300000000000000FE"
-"0000000300000000000000FF"
-"000000030000000000000100"
-"000000030000000000000101"
-"000000030000000000000102"
-"000000030000000000000103"
-"000000030000000000000104"
-"000000030000000000000105"
-"000000030000000000000106"
-"000000030000000000000107"
-"000000030000000000000108"
-"000000030000000000000109"
-"00000003000000000000010A"
-"00000003000000000000010B"
-"00000003000000000000010C"
-"00000003000000000000010D"
-"00000003000000000000010E"
-"00000003000000000000010F"
-"000000030000000000000110"
-"000000030000000000000111"
-"000000030000000000000112"
-"000000030000000000000113"
-"000000030000000000000114"
-"000000030000000000000115"
-"000000030000000000000116"
-"000000030000000000000117"
-"000000030000000000000118"
-"000000030000000000000119"
-"00000003000000000000011A"
-"00000003000000000000011B"
-"00000003000000000000011C"
-"00000003000000000000011D"
-"00000003000000000000011E"
-"00000003000000000000011F"
-"000000030000000000000120"
-"000000030000000000000121"
-"000000030000000000000122"
-"000000030000000000000123"
-"000000030000000000000124"
-"000000030000000000000125"
-"000000030000000000000126"
-"000000030000000000000127"
-"000000030000000000000128"
-"000000030000000000000129"
-"00000003000000000000012A"
-"00000003000000000000012B"
-"00000003000000000000012C"
-"00000003000000000000012D"
-"00000003000000000000012E"
-"00000003000000000000012F"
-"000000030000000000000130"
-"000000030000000000000131"
-"000000030000000000000132"
-"000000030000000000000133"
-"000000030000000000000134"
-"000000030000000000000135"
-"000000030000000000000136"
-"000000030000000000000137"
-"000000030000000000000138"
-"000000030000000000000139"
-"00000003000000000000013A"
-"00000003000000000000013B"
-"00000003000000000000013C"
-"00000003000000000000013D"
-"00000003000000000000013E"
-"00000003000000000000013F"
-"000000030000000000000140"
-"000000030000000000000141"
-"000000030000000000000142"
-"000000030000000000000143"
-"000000030000000000000144"
-"000000030000000000000145"
-"000000030000000000000146"
-"000000030000000000000147"
-"000000030000000000000148"
-"000000030000000000000149"
-"00000003000000000000014A"
-"00000003000000000000014B"
-"00000003000000000000014C"
-"00000003000000000000014D"
-"00000003000000000000014E"
-"00000003000000000000014F"
-"000000030000000000000150"
-"000000030000000000000151"
-"000000030000000000000152"
-"000000030000000000000153"
-"000000030000000000000154"
-"000000030000000000000155"
-"000000030000000000000156"
-"000000030000000000000157"
-"000000030000000000000158"
-"000000030000000000000159"
-"00000003000000000000015A"
-"00000003000000000000015B"
-"00000003000000000000015C"
-"00000003000000000000015D"
-"00000003000000000000015E"
-"00000003000000000000015F"
-"000000030000000000000160"
-"000000030000000000000161"
-"000000030000000000000162"
-"000000030000000000000163"
-"000000030000000000000164"
-"000000030000000000000165"
-"000000030000000000000166"
-"000000030000000000000167"
-"000000030000000000000168"
-"000000030000000000000169"
-"00000003000000000000016A"
-"00000003000000000000016B"
-"00000003000000000000016C"
-"00000003000000000000016D"
-"00000003000000000000016E"
-"00000003000000000000016F"
-"000000030000000000000170"
-"000000030000000000000171"
-"000000030000000000000172"
-"000000030000000000000173"
-"000000030000000000000174"
-"000000030000000000000175"
-"000000030000000000000176"
-"000000030000000000000177"
-"000000030000000000000178"
-"000000030000000000000179"
-"00000003000000000000017A"
-"00000003000000000000017B"
-"00000003000000000000017C"
-"00000003000000000000017D"
-"00000003000000000000017E"
-"00000003000000000000017F"
-"000000030000000000000180"
-"000000030000000000000181"
-"000000030000000000000182"
-"000000030000000000000183"
-"000000030000000000000184"
-"000000030000000000000185"
-"000000030000000000000186"
-"000000030000000000000187"
-"000000030000000000000188"
-"000000030000000000000189"
-"00000003000000000000018A"
-"00000003000000000000018B"
-"00000003000000000000018C"
-"00000003000000000000018D"
-"00000003000000000000018E"
-"00000003000000000000018F"
-"000000030000000000000190"
-"000000030000000000000191"
-"000000030000000000000192"
-"000000030000000000000193"
-"000000030000000000000194"
-"000000030000000000000195"
-"000000030000000000000196"
-"000000030000000000000197"
-"000000030000000000000198"
-"000000030000000000000199"
-"00000003000000000000019A"
-"00000003000000000000019B"
-"00000003000000000000019C"
-"00000003000000000000019D"
-"00000003000000000000019E"
-"00000003000000000000019F"
-"0000000300000000000001A0"
-"0000000300000000000001A1"
-"0000000300000000000001A2"
-"0000000300000000000001A3"
-"0000000300000000000001A4"
-"0000000300000000000001A5"
-"0000000300000000000001A6"
-"0000000300000000000001A7"
-"0000000300000000000001A8"
-"0000000300000000000001A9"
-"0000000300000000000001AA"
-"0000000300000000000001AB"
-"0000000300000000000001AC"
-"0000000300000000000001AD"
-"0000000300000000000001AE"
-"0000000300000000000001AF"
-"0000000300000000000001B0"
-"0000000300000000000001B1"
-"0000000300000000000001B2"
-"0000000300000000000001B3"
-"0000000300000000000001B4"
-"0000000300000000000001B5"
-"0000000300000000000001B6"
-"0000000300000000000001B7"
-"0000000300000000000001B8"
-"0000000300000000000001B9"
-"0000000300000000000001BA"
-"0000000300000000000001BB"
-"0000000300000000000001BC"
-"0000000300000000000001BD"
-"0000000300000000000001BE"
-"0000000300000000000001BF"
-"0000000300000000000001C0"
-"0000000300000000000001C1"
-"0000000300000000000001C2"
-"0000000300000000000001C3"
-"0000000300000000000001C4"
-"0000000300000000000001C5"
-"0000000300000000000001C6"
-"0000000300000000000001C7"
-"0000000300000000000001C8"
-"0000000300000000000001C9"
-"0000000300000000000001CA"
-"0000000300000000000001CB"
-"0000000300000000000001CC"
-"0000000300000000000001CD"
-"0000000300000000000001CE"
-"0000000300000000000001CF"
-"0000000300000000000001D0"
-"0000000300000000000001D1"
-"0000000300000000000001D2"
-"0000000300000000000001D3"
-"0000000300000000000001D4"
-"0000000300000000000001D5"
-"0000000300000000000001D6"
-"0000000300000000000001D7"
-"0000000300000000000001D8"
-"0000000300000000000001D9"
-"0000000300000000000001DA"
-"0000000300000000000001DB"
-"0000000300000000000001DC"
-"0000000300000000000001DD"
-"0000000300000000000001DE"
-"0000000300000000000001DF"
-"0000000300000000000001E0"
-"0000000300000000000001E1"
-"0000000300000000000001E2"
-"0000000300000000000001E3"
-"0000000300000000000001E4"
-"0000000300000000000001E5"
-"0000000300000000000001E6"
-"0000000300000000000001E7"
-"0000000300000000000001E8"
-"0000000300000000000001E9"
-"0000000300000000000001EA"
-"0000000300000000000001EB"
-"0000000300000000000001EC"
-"0000000300000000000001ED"
-"0000000300000000000001EE"
-"0000000300000000000001EF"
-"0000000300000000000001F0"
-"0000000300000000000001F1"
-"0000000300000000000001F2"
-"0000000300000000000001F3"
-"0000000300000000000001F4"
-"0000000300000000000001F5"
-"0000000300000000000001F6"
-"0000000300000000000001F7"
-"0000000300000000000001F8"
-"0000000300000000000001F9"
-"0000000300000000000001FA"
-"0000000300000000000001FB"
-"0000000300000000000001FC"
-"0000000300000000000001FD"
-"0000000300000000000001FE"
-"0000000300000000000001FF"
-"000000030000000000000200"
-"000000030000000000000201"
-"000000030000000000000202"
-"000000030000000000000203"
-"000000030000000000000204"
-"000000030000000000000205"
-"000000030000000000000206"
-"000000030000000000000207"
-"000000030000000000000208"
-"000000030000000000000209"
-"00000003000000000000020A"
-"00000003000000000000020B"
-"00000003000000000000020C"
-"00000003000000000000020D"
-"00000003000000000000020E"
-"00000003000000000000020F"
-"000000030000000000000210"
-"000000030000000000000211"
-"000000030000000000000212"
-"000000030000000000000213"
-"000000030000000000000214"
-"000000030000000000000215"
-"000000030000000000000216"
-"000000030000000000000217"
-"000000030000000000000218"
-"000000030000000000000219"
-"00000003000000000000021A"
-"00000003000000000000021B"
-"00000003000000000000021C"
-"00000003000000000000021D"
-"00000003000000000000021E"
-"00000003000000000000021F"
-"000000030000000000000220"
-"000000030000000000000221"
-"000000030000000000000222"
-"000000030000000000000223"
-"000000030000000000000224"
-"000000030000000000000225"
-"000000030000000000000226"
-"000000030000000000000227"
-"000000030000000000000228"
-"000000030000000000000229"
-"00000003000000000000022A"
-"00000003000000000000022B"
-"00000003000000000000022C"
-"00000003000000000000022D"
-"00000003000000000000022E"
-"00000003000000000000022F"
-"000000030000000000000230"
-"000000030000000000000231"
-"000000030000000000000232"
-"000000030000000000000233"
-"000000030000000000000234"
-"000000030000000000000235"
-"000000030000000000000236"
-"000000030000000000000237"
-"000000030000000000000238"
-"000000030000000000000239"
-"00000003000000000000023A"
-"00000003000000000000023B"
-"00000003000000000000023C"
-"00000003000000000000023D"
-"00000003000000000000023E"
-"00000003000000000000023F"
-"000000030000000000000240"
-"000000030000000000000241"
-"000000030000000000000242"
-"000000030000000000000243"
-"000000030000000000000244"
-"000000030000000000000245"
-"000000030000000000000246"
-"000000030000000000000247"
-"000000030000000000000248"
-"000000030000000000000249"
-"00000003000000000000024A"
-"00000003000000000000024B"
-"00000003000000000000024C"
-"00000003000000000000024D"
-"00000003000000000000024E"
-"00000003000000000000024F"
-"000000030000000000000250"
-"000000030000000000000251"
-"000000030000000000000252"
-"000000030000000000000253"
-"000000030000000000000254"
-"000000030000000000000255"
-"000000030000000000000256"
-"000000030000000000000257"
-"000000030000000000000258"
-"000000030000000000000259"
-"00000003000000000000025A"
-"00000003000000000000025B"
-"00000003000000000000025C"
-"00000003000000000000025D"
-"00000003000000000000025E"
-"00000003000000000000025F"
-"000000030000000000000260"
-"000000030000000000000261"
-"000000030000000000000262"
-"000000030000000000000263"
-"000000030000000000000264"
-"000000030000000000000265"
-"000000030000000000000266"
-"000000030000000000000267"
-"000000030000000000000268"
-"000000030000000000000269"
-"00000003000000000000026A"
-"00000003000000000000026B"
-"00000003000000000000026C"
-"00000003000000000000026D"
-"00000003000000000000026E"
-"00000003000000000000026F"
-"000000030000000000000270"
-"000000030000000000000271"
-"000000030000000000000272"
-"000000030000000000000273"
-"000000030000000000000274"
-"000000030000000000000275"
-"000000030000000000000276"
-"000000030000000000000277"
-"000000030000000000000278"
-"000000030000000000000279"
-"00000003000000000000027A"
-"00000003000000000000027B"
-"00000003000000000000027C"
-"00000003000000000000027D"
-"00000003000000000000027E"
-"00000003000000000000027F"
-"000000030000000000000280"
-"000000030000000000000281"
-"000000030000000000000282"
-"000000030000000000000283"
-"000000030000000000000284"
-"000000030000000000000285"
-"000000030000000000000286"
-"000000030000000000000287"
-"000000030000000000000288"
-"000000030000000000000289"
-"00000003000000000000028A"
-"00000003000000000000028B"
-"00000003000000000000028C"
-"00000003000000000000028D"
-"00000003000000000000028E"
-"00000003000000000000028F"
-"000000030000000000000290"
-"000000030000000000000291"
-"000000030000000000000292"
-"000000030000000000000293"
-"000000030000000000000294"
-"000000030000000000000295"
-"000000030000000000000296"
-"000000030000000000000297"
-"000000030000000000000298"
-"000000030000000000000299"
-"00000003000000000000029A"
-"00000003000000000000029B"
-"00000003000000000000029C"
-"00000003000000000000029D"
-"00000003000000000000029E"
-"00000003000000000000029F"
-"0000000300000000000002A0"
-"0000000300000000000002A1"
-"0000000300000000000002A2"
-"0000000300000000000002A3"
-"0000000300000000000002A4"
-"0000000300000000000002A5"
-"0000000300000000000002A6"
-"0000000300000000000002A7"
-"0000000300000000000002A8"
-"0000000300000000000002A9"
-"0000000300000000000002AA"
-"0000000300000000000002AB"
-"0000000300000000000002AC"
-"0000000300000000000002AD"
-"0000000300000000000002AE"
-"0000000300000000000002AF"
-"0000000300000000000002B0"
-"0000000300000000000002B1"
-"0000000300000000000002B2"
-"0000000300000000000002B3"
-"0000000300000000000002B4"
-"0000000300000000000002B5"
-"0000000300000000000002B6"
-"0000000300000000000002B7"
-"0000000300000000000002B8"
-"0000000300000000000002B9"
-"0000000300000000000002BA"
-"0000000300000000000002BB"
-"0000000300000000000002BC"
-"0000000300000000000002BD"
-"0000000300000000000002BE"
-"0000000300000000000002BF"
-"0000000300000000000002C0"
-"0000000300000000000002C1"
-"0000000300000000000002C2"
-"0000000300000000000002C3"
-"0000000300000000000002C4"
-"0000000300000000000002C5"
-"0000000300000000000002C6"
-"0000000300000000000002C7"
-"0000000300000000000002C8"
-"0000000300000000000002C9"
-"0000000300000000000002CA"
-"0000000300000000000002CB"
-"0000000300000000000002CC"
-"0000000300000000000002CD"
-"0000000300000000000002CE"
-"0000000300000000000002CF"
-"0000000300000000000002D0"
-"0000000300000000000002D1"
-"0000000300000000000002D2"
-"0000000300000000000002D3"
-"0000000300000000000002D4"
-"0000000300000000000002D5"
-"0000000300000000000002D6"
-"0000000300000000000002D7"
-"0000000300000000000002D8"
-"0000000300000000000002D9"
-"0000000300000000000002DA"
-"0000000300000000000002DB"
-"0000000300000000000002DC"
-"0000000300000000000002DD"
-"0000000300000000000002DE"
-"0000000300000000000002DF"
-"0000000300000000000002E0"
-"0000000300000000000002E1"
-"0000000300000000000002E2"
-"0000000300000000000002E3"
-"0000000300000000000002E4"
-"0000000300000000000002E5"
-"0000000300000000000002E6"
-"0000000300000000000002E7"
-"0000000300000000000002E8"
-"0000000300000000000002E9"
-"0000000300000000000002EA"
-"0000000300000000000002EB"
-"0000000300000000000002EC"
-"0000000300000000000002ED"
-"0000000300000000000002EE"
-"0000000300000000000002EF"
-"0000000300000000000002F0"
-"0000000300000000000002F1"
-"0000000300000000000002F2"
-"0000000300000000000002F3"
-"0000000300000000000002F4"
-"0000000300000000000002F5"
-"0000000300000000000002F6"
-"0000000300000000000002F7"
-"0000000300000000000002F8"
-"0000000300000000000002F9"
-"0000000300000000000002FA"
-"0000000300000000000002FB"
-"0000000300000000000002FC"
-"0000000300000000000002FD"
-"0000000300000000000002FE"
-"0000000300000000000002FF"
-"000000030000000000000300"
-"000000030000000000000301"
-"000000030000000000000302"
-"000000030000000000000303"
-"000000030000000000000304"
-"000000030000000000000305"
-"000000030000000000000306"
-"000000030000000000000307"
-"000000030000000000000308"
-"000000030000000000000309"
-"00000003000000000000030A"
-"00000003000000000000030B"
-"00000003000000000000030C"
-"00000003000000000000030D"
-"00000003000000000000030E"
-"00000003000000000000030F"
-"000000030000000000000310"
-"000000030000000000000311"
-"000000030000000000000312"
-"000000030000000000000313"
-"000000030000000000000314"
-"000000030000000000000315"
-"000000030000000000000316"
-"000000030000000000000317"
-"000000030000000000000318"
-"000000030000000000000319"
-"00000003000000000000031A"
-"00000003000000000000031B"
-"00000003000000000000031C"
-"00000003000000000000031D"
-"00000003000000000000031E"
-"00000003000000000000031F"
-"000000030000000000000320"
-"000000030000000000000321"
-"000000030000000000000322"
-"000000030000000000000323"
-"000000030000000000000324"
-"000000030000000000000325"
-"000000030000000000000326"
-"000000030000000000000327"
-"000000030000000000000328"
-"000000030000000000000329"
-"00000003000000000000032A"
-"00000003000000000000032B"
-"00000003000000000000032C"
-"00000003000000000000032D"
-"00000003000000000000032E"
-"00000003000000000000032F"
-"000000030000000000000330"
-"000000030000000000000331"
-"000000030000000000000332"
-"000000030000000000000333"
-"000000030000000000000334"
-"000000030000000000000335"
-"000000030000000000000336"
-"000000030000000000000337"
-"000000030000000000000338"
-"000000030000000000000339"
-"00000003000000000000033A"
-"00000003000000000000033B"
-"00000003000000000000033C"
-"00000003000000000000033D"
-"00000003000000000000033E"
-"00000003000000000000033F"
-"000000030000000000000340"
-"000000030000000000000341"
-"000000030000000000000342"
-"000000030000000000000343"
-"000000030000000000000344"
-"000000030000000000000345"
-"000000030000000000000346"
-"000000030000000000000347"
-"000000030000000000000348"
-"000000030000000000000349"
-"00000003000000000000034A"
-"00000003000000000000034B"
-"00000003000000000000034C"
-"00000003000000000000034D"
-"00000003000000000000034E"
-"00000003000000000000034F"
-"000000030000000000000350"
-"000000030000000000000351"
-"000000030000000000000352"
-"000000030000000000000353"
-"000000030000000000000354"
-"000000030000000000000355"
-"000000030000000000000356"
-"000000030000000000000357"
-"000000030000000000000358"
-"000000030000000000000359"
-"00000003000000000000035A"
-"00000003000000000000035B"
-"00000003000000000000035C"
-"00000003000000000000035D"
-"00000003000000000000035E"
-"00000003000000000000035F"
-"000000030000000000000360"
-"000000030000000000000361"
-"000000030000000000000362"
-"000000030000000000000363"
-"000000030000000000000364"
-"000000030000000000000365"
-"000000030000000000000366"
-"000000030000000000000367"
-"000000030000000000000368"
-"000000030000000000000369"
-"00000003000000000000036A"
-"00000003000000000000036B"
-"00000003000000000000036C"
-"00000003000000000000036D"
-"00000003000000000000036E"
-"00000003000000000000036F"
-"000000030000000000000370"
-"000000030000000000000371"
-"000000030000000000000372"
-"000000030000000000000373"
-"000000030000000000000374"
-"000000030000000000000375"
-"000000030000000000000376"
-"000000030000000000000377"
-"000000030000000000000378"
-"000000030000000000000379"
-"00000003000000000000037A"
-"00000003000000000000037B"
-"00000003000000000000037C"
-"00000003000000000000037D"
-"00000003000000000000037E"
-"00000003000000000000037F"
-"000000030000000000000380"
-"000000030000000000000381"
-"000000030000000000000382"
-"000000030000000000000383"
-"000000030000000000000384"
-"000000030000000000000385"
-"000000030000000000000386"
-"000000030000000000000387"
-"000000030000000000000388"
-"000000030000000000000389"
-"00000003000000000000038A"
-"00000003000000000000038B"
-"00000003000000000000038C"
-"00000003000000000000038D"
-"00000003000000000000038E"
-"00000003000000000000038F"
-"000000030000000000000390"
-"000000030000000000000391"
-"000000030000000000000392"
-"000000030000000000000393"
-"000000030000000000000394"
-"000000030000000000000395"
-"000000030000000000000396"
-"000000030000000000000397"
-"000000030000000000000398"
-"000000030000000000000399"
-"00000003000000000000039A"
-"00000003000000000000039B"
-"00000003000000000000039C"
-"00000003000000000000039D"
-"00000003000000000000039E"
-"00000003000000000000039F"
-"0000000300000000000003A0"
-"0000000300000000000003A1"
-"0000000300000000000003A2"
-"0000000300000000000003A3"
-"0000000300000000000003A4"
-"0000000300000000000003A5"
-"0000000300000000000003A6"
-"0000000300000000000003A7"
-"0000000300000000000003A8"
-"0000000300000000000003A9"
-"0000000300000000000003AA"
-"0000000300000000000003AB"
-"0000000300000000000003AC"
-"0000000300000000000003AD"
-"0000000300000000000003AE"
-"0000000300000000000003AF"
-"0000000300000000000003B0"
-"0000000300000000000003B1"
-"0000000300000000000003B2"
-"0000000300000000000003B3"
-"0000000300000000000003B4"
-"0000000300000000000003B5"
-"0000000300000000000003B6"
-"0000000300000000000003B7"
-"0000000300000000000003B8"
-"0000000300000000000003B9"
-"0000000300000000000003BA"
-"0000000300000000000003BB"
-"0000000300000000000003BC"
-"0000000300000000000003BD"
-"0000000300000000000003BE"
-"0000000300000000000003BF"
-"0000000300000000000003C0"
-"0000000300000000000003C1"
-"0000000300000000000003C2"
-"0000000300000000000003C3"
-"0000000300000000000003C4"
-"0000000300000000000003C5"
-"0000000300000000000003C6"
-"0000000300000000000003C7"
-"0000000300000000000003C8"
-"0000000300000000000003C9"
-"0000000300000000000003CA"
-"0000000300000000000003CB"
-"0000000300000000000003CC"
-"0000000300000000000003CD"
-"0000000300000000000003CE"
-"0000000300000000000003CF"
-"0000000300000000000003D0"
-"0000000300000000000003D1"
-"0000000300000000000003D2"
-"0000000300000000000003D3"
-"0000000300000000000003D4"
-"0000000300000000000003D5"
-"0000000300000000000003D6"
-"0000000300000000000003D7"
-"0000000300000000000003D8"
-"0000000300000000000003D9"
-"0000000300000000000003DA"
-"0000000300000000000003DB"
-"0000000300000000000003DC"
-"0000000300000000000003DD"
-"0000000300000000000003DE"
-"0000000300000000000003DF"
-"0000000300000000000003E0"
-"0000000300000000000003E1"
-"0000000300000000000003E2"
-"0000000300000000000003E3"
-"0000000300000000000003E4"
-"0000000300000000000003E5"
-"0000000300000000000003E6"
-"0000000300000000000003E7"
-"0000000300000000000003E8"
-"0000000300000000000003E9"
-"0000000300000000000003EA"
-"0000000300000000000003EB"
-"0000000300000000000003EC"
-"0000000300000000000003ED"
-"0000000300000000000003EE"
-"0000000300000000000003EF"
-"0000000300000000000003F0"
-"0000000300000000000003F1"
-"0000000300000000000003F2"
-"0000000300000000000003F3"
-"0000000300000000000003F4"
-"0000000300000000000003F5"
-"0000000300000000000003F6"
-"0000000300000000000003F7"
-"0000000300000000000003F8"
-"0000000300000000000003F9"
-"0000000300000000000003FA"
-"0000000300000000000003FB"
-"0000000300000000000003FC"
-"0000000300000000000003FD"
-"0000000300000000000003FE"
-"0000000300000000000003FF"
-"000000030000000000000400"
-"000000030000000000000401"
-"000000030000000000000402"
-"000000030000000000000403"
-"000000030000000000000404"
-"000000030000000000000405"
-"000000030000000000000406"
-"000000030000000000000407"
-"000000030000000000000408"
-"000000030000000000000409"
-"00000003000000000000040A"
-"00000003000000000000040B"
-"00000003000000000000040C"
-"00000003000000000000040D"
-"00000003000000000000040E"
-"00000003000000000000040F"
-"000000030000000000000410"
-"000000030000000000000411"
-"000000030000000000000412"
-"000000030000000000000413"
-"000000030000000000000414"
-"000000030000000000000415"
-"000000030000000000000416"
-"000000030000000000000417"
-"000000030000000000000418"
-"000000030000000000000419"
-"00000003000000000000041A"
-"00000003000000000000041B"
-"00000003000000000000041C"
-"00000003000000000000041D"
-"00000003000000000000041E"
-"00000003000000000000041F"
-"000000030000000000000420"
-"000000030000000000000421"
-"000000030000000000000422"
-"000000030000000000000423"
-"000000030000000000000424"
-"000000030000000000000425"
-"000000030000000000000426"
-"000000030000000000000427"
-"000000030000000000000428"
-"000000030000000000000429"
-"00000003000000000000042A"
-"00000003000000000000042B"
-"00000003000000000000042C"
-"00000003000000000000042D"
-"00000003000000000000042E"
-"00000003000000000000042F"
-"000000030000000000000430"
-"000000030000000000000431"
-"000000030000000000000432"
-"000000030000000000000433"
-"000000030000000000000434"
-"000000030000000000000435"
-"000000030000000000000436"
-"000000030000000000000437"
-"000000030000000000000438"
-"000000030000000000000439"
-"00000003000000000000043A"
-"00000003000000000000043B"
-"00000003000000000000043C"
-"00000003000000000000043D"
-"00000003000000000000043E"
-"00000003000000000000043F"
-"000000030000000000000440"
-"000000030000000000000441"
-"000000030000000000000442"
-"000000030000000000000443"
-"000000030000000000000444"
-"000000030000000000000445"
-"000000030000000000000446"
-"000000030000000000000447"
-"000000030000000000000448"
-"000000030000000000000449"
-"00000003000000000000044A"
-"00000003000000000000044B"
-"00000003000000000000044C"
-"00000003000000000000044D"
-"00000003000000000000044E"
-"00000003000000000000044F"
-"000000030000000000000450"
-"000000030000000000000451"
-"000000030000000000000452"
-"000000030000000000000453"
-"000000030000000000000454"
-"000000030000000000000455"
-"000000030000000000000456"
-"000000030000000000000457"
-"000000030000000000000458"
-"000000030000000000000459"
-"00000003000000000000045A"
-"00000003000000000000045B"
-"00000003000000000000045C"
-"00000003000000000000045D"
-"00000003000000000000045E"
-"00000003000000000000045F"
-"000000030000000000000460"
-"000000030000000000000461"
-"000000030000000000000462"
-"000000030000000000000463"
-"000000030000000000000464"
-"000000030000000000000465"
-"000000030000000000000466"
-"000000030000000000000467"
-"000000030000000000000468"
-"000000030000000000000469"
-"00000003000000000000046A"
-"00000003000000000000046B"
-"00000003000000000000046C"
-"00000003000000000000046D"
-"00000003000000000000046E"
-"00000003000000000000046F"
-"000000030000000000000470"
-"000000030000000000000471"
-"000000030000000000000472"
-"000000030000000000000473"
-"000000030000000000000474"
-"000000030000000000000475"
-"000000030000000000000476"
-"000000030000000000000477"
-"000000030000000000000478"
-"000000030000000000000479"
-"00000003000000000000047A"
-"00000003000000000000047B"
-"00000003000000000000047C"
-"00000003000000000000047D"
-"00000003000000000000047E"
-"00000003000000000000047F"
-"000000030000000000000480"
-"000000030000000000000481"
-"000000030000000000000482"
-"000000030000000000000483"
-"000000030000000000000484"
-"000000030000000000000485"
-"000000030000000000000486"
-"000000030000000000000487"
-"000000030000000000000488"
-"000000030000000000000489"
-"00000003000000000000048A"
-"00000003000000000000048B"
-"00000003000000000000048C"
-"00000003000000000000048D"
-"00000003000000000000048E"
-"00000003000000000000048F"
-"000000030000000000000490"
-"000000030000000000000491"
-"000000030000000000000492"
-"000000030000000000000493"
-"000000030000000000000494"
-"000000030000000000000495"
-"000000030000000000000496"
-"000000030000000000000497"
-"000000030000000000000498"
-"000000030000000000000499"
-"00000003000000000000049A"
-"00000003000000000000049B"
-"00000003000000000000049C"
-"00000003000000000000049D"
-"00000003000000000000049E"
-"00000003000000000000049F"
-"0000000300000000000004A0"
-"0000000300000000000004A1"
-"0000000300000000000004A2"
-"0000000300000000000004A3"
-"0000000300000000000004A4"
-"0000000300000000000004A5"
-"0000000300000000000004A6"
-"0000000300000000000004A7"
-"0000000300000000000004A8"
-"0000000300000000000004A9"
-"0000000300000000000004AA"
-"0000000300000000000004AB"
-"0000000300000000000004AC"
-"0000000300000000000004AD"
-"0000000300000000000004AE"
-"0000000300000000000004AF"
-"0000000300000000000004B0"
-"0000000300000000000004B1"
-"0000000300000000000004B2"
-"0000000300000000000004B3"
-"0000000300000000000004B4"
-"0000000300000000000004B5"
-"0000000300000000000004B6"
-"0000000300000000000004B7"
-"0000000300000000000004B8"
-"0000000300000000000004B9"
-"0000000300000000000004BA"
-"0000000300000000000004BB"
-"0000000300000000000004BC"
-"0000000300000000000004BD"
-"0000000300000000000004BE"
-"0000000300000000000004BF"
-"0000000300000000000004C0"
-"0000000300000000000004C1"
-"0000000300000000000004C2"
-"0000000300000000000004C3"
-"0000000300000000000004C4"
-"0000000300000000000004C5"
-"0000000300000000000004C6"
-"0000000300000000000004C7"
-"0000000300000000000004C8"
-"0000000300000000000004C9"
-"0000000300000000000004CA"
-"0000000300000000000004CB"
-"0000000300000000000004CC"
-"0000000300000000000004CD"
-"0000000300000000000004CE"
-"0000000300000000000004CF"
-"0000000300000000000004D0"
-"0000000300000000000004D1"
-"0000000300000000000004D2"
-"0000000300000000000004D3"
-"0000000300000000000004D4"
-"0000000300000000000004D5"
-"0000000300000000000004D6"
-"0000000300000000000004D7"
-"0000000300000000000004D8"
-"0000000300000000000004D9"
-"0000000300000000000004DA"
-"0000000300000000000004DB"
-"0000000300000000000004DC"
-"0000000300000000000004DD"
-"0000000300000000000004DE"
-"0000000300000000000004DF"
-"0000000300000000000004E0"
-"0000000300000000000004E1"
-"0000000300000000000004E2"
-"0000000300000000000004E3"
-"0000000300000000000004E4"
-"0000000300000000000004E5"
-"0000000300000000000004E6"
-"0000000300000000000004E7"
-"0000000300000000000004E8"
-"0000000300000000000004E9"
-"0000000300000000000004EA"
-"0000000300000000000004EB"
-"0000000300000000000004EC"
-"0000000300000000000004ED"
-"0000000300000000000004EE"
-"0000000300000000000004EF"
-"0000000300000000000004F0"
-"0000000300000000000004F1"
-"0000000300000000000004F2"
-"0000000300000000000004F3"
-"0000000300000000000004F4"
-"0000000300000000000004F5"
-"0000000300000000000004F6"
-"0000000300000000000004F7"
-"0000000300000000000004F8"
-"0000000300000000000004F9"
-"0000000300000000000004FA"
-"0000000300000000000004FB"
-"0000000300000000000004FC"
-"0000000300000000000004FD"
-"0000000300000000000004FE"
-"0000000300000000000004FF"
-"000000030000000000000500"
-"000000030000000000000501"
-"000000030000000000000502"
-"000000030000000000000503"
-"000000030000000000000504"
-"000000030000000000000505"
-"000000030000000000000506"
-"000000030000000000000507"
-"000000030000000000000508"
-"000000030000000000000509"
-"00000003000000000000050A"
-"00000003000000000000050B"
-"00000003000000000000050C"
-"00000003000000000000050D"
-"00000003000000000000050E"
-"00000003000000000000050F"
-"000000030000000000000510"
-"000000030000000000000511"
-"000000030000000000000512"
-"000000030000000000000513"
-"000000030000000000000514"
-"000000030000000000000515"
-"000000030000000000000516"
-"000000030000000000000517"
-"000000030000000000000518"
-"000000030000000000000519"
-"00000003000000000000051A"
-"00000003000000000000051B"
-"00000003000000000000051C"
-"00000003000000000000051D"
-"00000003000000000000051E"
-"00000003000000000000051F"
-"000000030000000000000520"
-"000000030000000000000521"
-"000000030000000000000522"
-"000000030000000000000523"
-"000000030000000000000524"
-"000000030000000000000525"
-"000000030000000000000526"
-"000000030000000000000527"
-"000000030000000000000528"
-"000000030000000000000529"
-"00000003000000000000052A"
-"00000003000000000000052B"
-"00000003000000000000052C"
-"00000003000000000000052D"
-"00000003000000000000052E"
-"00000003000000000000052F"
-"000000030000000000000530"
-"000000030000000000000531"
-"000000030000000000000532"
-"000000030000000000000533"
-"000000030000000000000534"
-"000000030000000000000535"
-"000000030000000000000536"
-"000000030000000000000537"
-"000000030000000000000538"
-"000000030000000000000539"
-"00000003000000000000053A"
-"00000003000000000000053B"
-"00000003000000000000053C"
-"00000003000000000000053D"
-"00000003000000000000053E"
-"00000003000000000000053F"
-"000000030000000000000540"
-"000000030000000000000541"
-"000000030000000000000542"
-"000000030000000000000543"
-"000000030000000000000544"
-"000000030000000000000545"
-"000000030000000000000546"
-"000000030000000000000547"
-"000000030000000000000548"
-"000000030000000000000549"
-"00000003000000000000054A"
-"00000003000000000000054B"
-"00000003000000000000054C"
-"00000003000000000000054D"
-"00000003000000000000054E"
-"00000003000000000000054F"
-"000000030000000000000550"
-"000000030000000000000551"
-"000000030000000000000552"
-"000000030000000000000553"
-"000000030000000000000554"
-"000000030000000000000555"
-"000000030000000000000556"
-"000000030000000000000557"
-"000000030000000000000558"
-"000000030000000000000559"
-"00000003000000000000055A"
-"00000003000000000000055B"
-"00000003000000000000055C"
-"00000003000000000000055D"
-"00000003000000000000055E"
-"00000003000000000000055F"
-"000000030000000000000560"
-"000000030000000000000561"
-"000000030000000000000562"
-"000000030000000000000563"
-"000000030000000000000564"
-"000000030000000000000565"
-"000000030000000000000566"
-"000000030000000000000567"
-"000000030000000000000568"
-"000000030000000000000569"
-"00000003000000000000056A"
-"00000003000000000000056B"
-"00000003000000000000056C"
-"00000003000000000000056D"
-"00000003000000000000056E"
-"00000003000000000000056F"
-"000000030000000000000570"
-"000000030000000000000571"
-"000000030000000000000572"
-"000000030000000000000573"
-"000000030000000000000574"
-"000000030000000000000575"
-"000000030000000000000576"
-"000000030000000000000577"
-"000000030000000000000578"
-"000000030000000000000579"
-"00000003000000000000057A"
-"00000003000000000000057B"
-"00000003000000000000057C"
-"00000003000000000000057D"
-"00000003000000000000057E"
-"00000003000000000000057F"
-"000000030000000000000580"
-"000000030000000000000581"
-"000000030000000000000582"
-"000000030000000000000583"
-"000000030000000000000584"
-"000000030000000000000585"
-"000000030000000000000586"
-"000000030000000000000587"
-"000000030000000000000588"
-"000000030000000000000589"
-"00000003000000000000058A"
-"00000003000000000000058B"
-"00000003000000000000058C"
-"00000003000000000000058D"
-"00000003000000000000058E"
-"00000003000000000000058F"
-"000000030000000000000590"
-"000000030000000000000591"
-"000000030000000000000592"
-"000000030000000000000593"
-"000000030000000000000594"
-"000000030000000000000595"
-"000000030000000000000596"
-"000000030000000000000597"
-"000000030000000000000598"
-"000000030000000000000599"
-"00000003000000000000059A"
-"00000003000000000000059B"
-"00000003000000000000059C"
-"00000003000000000000059D"
-"00000003000000000000059E"
-"00000003000000000000059F"
-"0000000300000000000005A0"
-"0000000300000000000005A1"
-"0000000300000000000005A2"
-"0000000300000000000005A3"
-"0000000300000000000005A4"
-"0000000300000000000005A5"
-"0000000300000000000005A6"
-"0000000300000000000005A7"
-"0000000300000000000005A8"
-"0000000300000000000005A9"
-"0000000300000000000005AA"
-"0000000300000000000005AB"
-"0000000300000000000005AC"
-"0000000300000000000005AD"
-"0000000300000000000005AE"
-"0000000300000000000005AF"
-"0000000300000000000005B0"
-"0000000300000000000005B1"
-"0000000300000000000005B2"
-"0000000300000000000005B3"
-"0000000300000000000005B4"
-"0000000300000000000005B5"
-"0000000300000000000005B6"
-"0000000300000000000005B7"
-"0000000300000000000005B8"
-"0000000300000000000005B9"
-"0000000300000000000005BA"
-"0000000300000000000005BB"
-"0000000300000000000005BC"
-"0000000300000000000005BD"
-"0000000300000000000005BE"
-"0000000300000000000005BF"
-"0000000300000000000005C0"
-"0000000300000000000005C1"
-"0000000300000000000005C2"
-"0000000300000000000005C3"
-"0000000300000000000005C4"
-"0000000300000000000005C5"
-"0000000300000000000005C6"
-"0000000300000000000005C7"
-"0000000300000000000005C8"
-"0000000300000000000005C9"
-"0000000300000000000005CA"
-"0000000300000000000005CB"
-"0000000300000000000005CC"
-"0000000300000000000005CD"
-"0000000300000000000005CE"
-"0000000300000000000005CF"
-"0000000300000000000005D0"
-"0000000300000000000005D1"
-"0000000300000000000005D2"
-"0000000300000000000005D3"
-"0000000300000000000005D4"
-"0000000300000000000005D5"
-"0000000300000000000005D6"
-"0000000300000000000005D7"
-"0000000300000000000005D8"
-"0000000300000000000005D9"
-"0000000300000000000005DA"
-"0000000300000000000005DB"
-"0000000300000000000005DC"
-"0000000300000000000005DD"
-"0000000300000000000005DE"
-"0000000300000000000005DF"
-"0000000300000000000005E0"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
new file mode 100644
index 0000000..5db606c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
@@ -0,0 +1 @@
+{ "count": 25 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
new file mode 100644
index 0000000..7660e7e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
@@ -0,0 +1,25 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
new file mode 100644
index 0000000..5db606c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
@@ -0,0 +1 @@
+{ "count": 25 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
new file mode 100644
index 0000000..7643986
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
@@ -0,0 +1,25 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.3.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.3.json
new file mode 100644
index 0000000..58de662
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.3.json
@@ -0,0 +1 @@
+"BAR=BAZ"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.4.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.4.json
new file mode 100644
index 0000000..f73df98
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.4.json
@@ -0,0 +1 @@
+"BAZ"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm
new file mode 100644
index 0000000..1a31db8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm
@@ -0,0 +1,2 @@
+{ "t0_unique1": 1891, "t1_unique1": 1891, "t2_unique1": 1891 }
+{ "t0_unique1": 8800, "t1_unique1": 8800, "t2_unique1": 8800 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
index 9abda4f..7edd3eb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
@@ -1,6 +1,6 @@
{ "CanonicalName": "A", "DisplayName": "A", "NameParts": [ "A" ] }
-{ "CanonicalName": "B.C", "DisplayName": "B.C", "NameParts": [ "B", "C" ] }
-{ "CanonicalName": "C@.D@.E", "DisplayName": "`C.D.E`", "NameParts": [ "C.D.E" ] }
+{ "CanonicalName": "B/C", "DisplayName": "B.C", "NameParts": [ "B", "C" ] }
+{ "CanonicalName": "C.D.E", "DisplayName": "`C.D.E`", "NameParts": [ "C.D.E" ] }
{ "CanonicalName": "Default", "DisplayName": "Default", "NameParts": [ "Default" ] }
{ "CanonicalName": "Metadata", "DisplayName": "Metadata", "NameParts": [ "Metadata" ] }
-{ "CanonicalName": "a-A.b_B.c$C.z@.Z", "DisplayName": "`a-A`.b_B.c$C.`z.Z`", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
+{ "CanonicalName": "a-A/b_B/c$C/z.Z", "DisplayName": "`a-A`.b_B.c$C.`z.Z`", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/in_let/in_let.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/in_let/in_let.8.adm
new file mode 100644
index 0000000..f63d51b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/in_let/in_let.8.adm
@@ -0,0 +1,2 @@
+{ "cid": "c1", "pid": "p2", "ts": 2000 }
+{ "cid": "c2", "pid": "p2", "ts": 2100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/synonym/synonym-03/synonym-03.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/synonym/synonym-03/synonym-03.2.adm
new file mode 100644
index 0000000..32200e3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/synonym/synonym-03/synonym-03.2.adm
@@ -0,0 +1,2 @@
+{ "DataverseName": "test2", "SynonymName": "md_syn_1", "ObjectDataverseName": "Metadata", "ObjectName": "Synonym" }
+{ "DataverseName": "test2", "SynonymName": "md_syn_2", "ObjectDataverseName": "test2", "ObjectName": "md_syn_1" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
index 4e1d5b2..35bec85 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -57,7 +57,7 @@
result[0].append(self.next_tuple(*arg, key=self.mid))
File "entrypoint.py", line 99, in next_tuple
return self.wrapped_fns[key](*args)
- File "site-packages/roundtrip.py", line 28, in warning
+ File "site-packages/roundtrip.py", line 29, in warning
raise ArithmeticError("oof")
ArithmeticError: oof
(in line 28, at column 1)</expected-warn>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 6492ef9..6e212b1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -4855,7 +4855,7 @@
<test-case FilePath="dml">
<compilation-unit name="upsert-dataset-with-meta">
<output-dir compare="Text">upsert-dataset-with-meta</output-dir>
- <expected-error>upsert into dataset is not supported on Datasets with Meta records</expected-error>
+ <expected-error>upsert into dataset is not supported on datasets with meta record</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
@@ -4881,13 +4881,13 @@
<test-case FilePath="dml">
<compilation-unit name="delete-dataset-with-meta">
<output-dir compare="Text">delete-dataset-with-meta</output-dir>
- <expected-error>delete from dataset is not supported on Datasets with Meta records</expected-error>
+ <expected-error>delete from dataset is not supported on datasets with meta records</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
<compilation-unit name="insert-dataset-with-meta">
<output-dir compare="Text">insert-dataset-with-meta</output-dir>
- <expected-error>insert into dataset is not supported on Datasets with Meta records</expected-error>
+ <expected-error>insert into dataset is not supported on datasets with meta records</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
@@ -6697,7 +6697,7 @@
<output-dir compare="Text">partition-by-nonexistent-field</output-dir>
<expected-error>Field "id" is not found</expected-error>
<expected-error>Cannot find dataset with name testds in dataverse test</expected-error>
- <expected-error>Cannot find dataset testds in dataverse test nor an alias with name testds!</expected-error>
+ <expected-error>Cannot find dataset testds in dataverse test nor an alias with name testds</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="misc">
@@ -10890,6 +10890,11 @@
<expected-error>ASX1063: Cannot find dataverse with name UNKNOWN_DATAVERSE</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="synonym">
+ <compilation-unit name="synonym-03">
+ <output-dir compare="Text">synonym-03</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="tokenizers">
<test-case FilePath="tokenizers">
@@ -12378,26 +12383,26 @@
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-1">
<output-dir compare="Text">bad-function-ddl-1</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages!</expected-error>
- <expected-error>Cannot find dataset TweetMessages2 in dataverse experiments2 nor an alias with name TweetMessages2!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages</expected-error>
+ <expected-error>Cannot find dataset TweetMessages2 in dataverse experiments2 nor an alias with name TweetMessages2</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-2">
<output-dir compare="Text">bad-function-ddl-2</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experiments2 nor an alias with name TweetMessages!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments2 nor an alias with name TweetMessages</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-3">
<output-dir compare="Text">bad-function-ddl-3</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-4">
<output-dir compare="Text">bad-function-ddl-4</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experients nor an alias with name TweetMessages!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experients nor an alias with name TweetMessages</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
@@ -12416,13 +12421,13 @@
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-7">
<output-dir compare="Text">bad-function-ddl-7</output-dir>
- <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes!</expected-error>
+ <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-8">
<output-dir compare="Text">bad-function-ddl-8</output-dir>
- <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes!</expected-error>
+ <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
@@ -12910,7 +12915,7 @@
<test-case FilePath="load">
<compilation-unit name="dataset-with-meta">
<output-dir compare="Text">dataset-with-meta</output-dir>
- <expected-error>ASX1079: Compilation error: DatasetWithMeta: load dataset is not supported on Datasets with Meta records (in line 27, at column 1)</expected-error>
+ <expected-error>ASX1079: Compilation error: DatasetWithMeta: load dataset is not supported on datasets with meta records (in line 27, at column 1)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="load">
@@ -13278,7 +13283,7 @@
<test-case FilePath="meta">
<compilation-unit name="query_dataset_with_meta_failure">
<output-dir compare="Text">query_dataset_with_meta_failure</output-dir>
- <expected-error>ASX1079: Compilation error: Cannot resolve ambiguous meta function call. There are more than one dataset choice! (in line 24, at column 7)</expected-error>
+ <expected-error>ASX1079: Compilation error: Cannot resolve ambiguous meta function call. There are more than one dataset choice (in line 24, at column 7)</expected-error>
</compilation-unit>
</test-case>
</test-group>
@@ -13393,6 +13398,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="leftouterjoin">
+ <compilation-unit name="query-ASTERIXDB-2857">
+ <output-dir compare="Text">query-ASTERIXDB-2857</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
<compilation-unit name="right_branch_opt_1">
<output-dir compare="Text">right_branch_opt_1</output-dir>
</compilation-unit>
@@ -14091,11 +14101,6 @@
<output-dir compare="Text">length</output-dir>
</compilation-unit>
</test-case>
- <test-case FilePath="binary">
- <compilation-unit name="query_id">
- <output-dir compare="Text">query_id</output-dir>
- </compilation-unit>
- </test-case>
</test-group>
<test-group name="unnest">
<test-case FilePath="unnest">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IIdentifierMapper.java
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
copy to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IIdentifierMapper.java
index 056d96e..8687239 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IIdentifierMapper.java
@@ -17,8 +17,11 @@
* under the License.
*/
-use tpch;
+package org.apache.asterix.common.api;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
+@FunctionalInterface
+public interface IIdentifierMapper {
+
+ String map(String identifier);
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 8064c05..5538a0a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -55,6 +55,10 @@
LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
"The memory budget (in bytes) for an inverted-index-search operator instance in a partition"),
+ COMPILER_EXTERNALSCANMEMORY(
+ INTEGER_BYTE_UNIT,
+ StorageUtil.getIntSizeInBytes(8, KILOBYTE),
+ "The memory budget (in bytes) for an external scan operator instance in a partition"),
COMPILER_FRAMESIZE(
INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
@@ -83,7 +87,7 @@
COMPILER_EXTERNAL_FIELD_PUSHDOWN(
BOOLEAN,
AlgebricksConfig.EXTERNAL_FIELD_PUSHDOWN_DEFAULT,
- "Enable pushdown of field accesses to the external dataset data-scan operator"),
+ "Enable pushdown of field accesses to the external data-scan operator"),
COMPILER_SUBPLAN_MERGE(
BOOLEAN,
AlgebricksConfig.SUBPLAN_MERGE_DEFAULT,
@@ -130,6 +134,11 @@
public Object defaultValue() {
return defaultValue;
}
+
+ @Override
+ public boolean hidden() {
+ return this == COMPILER_EXTERNALSCANMEMORY;
+ }
}
public static final String COMPILER_SORTMEMORY_KEY = Option.COMPILER_SORTMEMORY.ini();
@@ -162,6 +171,8 @@
public static final String COMPILER_ARRAYINDEX_KEY = Option.COMPILER_ARRAYINDEX.ini();
+ public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@ -231,4 +242,8 @@
public boolean isArrayIndex() {
return accessor.getBoolean(Option.COMPILER_ARRAYINDEX);
}
+
+ public int getExternalScanMemorySize() {
+ return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
index 9bef65d..46258bf 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
@@ -50,7 +50,8 @@
POSITIVE_INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(200, StorageUtil.StorageUnit.MEGABYTE),
"The maximum accepted web request size in bytes"),
- REQUESTS_ARCHIVE_SIZE(NONNEGATIVE_INTEGER, 50, "The maximum number of archived requests to maintain");
+ REQUESTS_ARCHIVE_SIZE(NONNEGATIVE_INTEGER, 50, "The maximum number of archived requests to maintain"),
+ LIBRARY_DEPLOY_TIMEOUT(POSITIVE_INTEGER, 1800, "Timeout to upload a UDF in seconds");
private final IOptionType type;
private final Object defaultValue;
@@ -76,6 +77,7 @@
case LOG_LEVEL:
case MAX_WAIT_ACTIVE_CLUSTER:
case MAX_WEB_REQUEST_SIZE:
+ case LIBRARY_DEPLOY_TIMEOUT:
return Section.COMMON;
case CC_JAVA_OPTS:
case NC_JAVA_OPTS:
@@ -149,4 +151,8 @@
return accessor.getInt(Option.REQUESTS_ARCHIVE_SIZE);
}
+ public int getLibraryDeployTimeout() {
+ return accessor.getInt(Option.LIBRARY_DEPLOY_TIMEOUT);
+ }
+
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index 81974fa..d498d69 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -74,6 +74,9 @@
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation());
boolean arrayIndex = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_ARRAYINDEX_KEY,
compilerProperties.isArrayIndex());
+ int externalScanBufferSize = getExternalScanBufferSize(
+ (String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
+ compilerProperties.getExternalScanMemorySize(), sourceLoc);
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
@@ -91,9 +94,21 @@
physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
physOptConf.setMinMemoryAllocation(minMemoryAllocation);
physOptConf.setArrayIndexEnabled(arrayIndex);
+ physOptConf.setExternalScanBufferSize(externalScanBufferSize);
return physOptConf;
}
+ private static int getExternalScanBufferSize(String externalScanMemorySizeParameter,
+ int compilerExternalScanMemorySize, SourceLocation sourceLoc) throws AsterixException {
+ IOptionType<Integer> intByteParser = OptionTypes.INTEGER_BYTE_UNIT;
+ try {
+ return externalScanMemorySizeParameter != null ? intByteParser.parse(externalScanMemorySizeParameter)
+ : compilerExternalScanMemorySize;
+ } catch (IllegalArgumentException e) {
+ throw AsterixException.create(ErrorCode.COMPILATION_ERROR, sourceLoc, e.getMessage());
+ }
+ }
+
public static int getSortNumFrames(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig,
SourceLocation sourceLoc) throws AlgebricksException {
return getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index fc33b1a..12c9c68 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -18,8 +18,8 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN;
import static org.apache.hyracks.control.common.config.OptionTypes.DOUBLE;
-import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
@@ -28,6 +28,7 @@
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
@@ -49,16 +50,18 @@
STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)),
STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(POSITIVE_INTEGER, 2),
STORAGE_MEMORYCOMPONENT_FLUSH_THRESHOLD(DOUBLE, 0.9d),
- STORAGE_MEMORYCOMPONENT_MAX_CONCURRENT_FLUSHES(INTEGER, 0),
+ STORAGE_MEMORYCOMPONENT_MAX_SCHEDULED_FLUSHES(NONNEGATIVE_INTEGER, 0),
STORAGE_FILTERED_MEMORYCOMPONENT_MAX_SIZE(LONG_BYTE_UNIT, 0L),
STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d),
STORAGE_COMPRESSION_BLOCK(STRING, "snappy"),
STORAGE_DISK_FORCE_BYTES(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(16, MEGABYTE)),
STORAGE_IO_SCHEDULER(STRING, "greedy"),
STORAGE_WRITE_RATE_LIMIT(LONG_BYTE_UNIT, 0l),
- STORAGE_MAX_RUNNING_FLUSHES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
+ STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
STORAGE_MAX_SCHEDULED_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 8),
- STORAGE_MAX_RUNNING_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 2);
+ STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
+ STORAGE_GLOBAL_CLEANUP(BOOLEAN, true),
+ STORAGE_GLOBAL_CLEANUP_TIMEOUT(POSITIVE_INTEGER, (int) TimeUnit.MINUTES.toSeconds(10));
private final IOptionType interpreter;
private final Object defaultValue;
@@ -73,6 +76,8 @@
switch (this) {
case STORAGE_COMPRESSION_BLOCK:
case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE:
+ case STORAGE_GLOBAL_CLEANUP:
+ case STORAGE_GLOBAL_CLEANUP_TIMEOUT:
return Section.COMMON;
default:
return Section.NC;
@@ -96,8 +101,8 @@
return "The page size in bytes for pages allocated to memory components";
case STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS:
return "The number of memory components to be used per lsm index";
- case STORAGE_MEMORYCOMPONENT_MAX_CONCURRENT_FLUSHES:
- return "The maximum number of concurrent flush operations. 0 means that the value will be "
+ case STORAGE_MEMORYCOMPONENT_MAX_SCHEDULED_FLUSHES:
+ return "The maximum number of scheduled flush operations. 0 means that the value will be "
+ "calculated as the number of partitions";
case STORAGE_MEMORYCOMPONENT_FLUSH_THRESHOLD:
return "The memory usage threshold when memory components should be flushed";
@@ -114,12 +119,16 @@
return "The number of bytes before each disk force (fsync)";
case STORAGE_IO_SCHEDULER:
return "The I/O scheduler for LSM flush and merge operations";
- case STORAGE_MAX_RUNNING_FLUSHES_PER_PARTITION:
- return "The maximum number of running flushes per partition (0 means unlimited)";
+ case STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION:
+ return "The maximum number of concurrently executed flushes per partition (0 means unlimited)";
case STORAGE_MAX_SCHEDULED_MERGES_PER_PARTITION:
return "The maximum number of scheduled merges per partition (0 means unlimited)";
- case STORAGE_MAX_RUNNING_MERGES_PER_PARTITION:
- return "The maximum number of running merges per partition (0 means unlimited)";
+ case STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION:
+ return "The maximum number of concurrently executed merges per partition (0 means unlimited)";
+ case STORAGE_GLOBAL_CLEANUP:
+ return "Indicates whether or not global storage cleanup is performed";
+ case STORAGE_GLOBAL_CLEANUP_TIMEOUT:
+ return "The maximum time to wait for nodes to respond to global storage cleanup requests";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -139,6 +148,11 @@
public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) {
return null;
}
+
+ @Override
+ public boolean hidden() {
+ return this == STORAGE_GLOBAL_CLEANUP;
+ }
}
public static final long MAX_HEAP_BYTES = Runtime.getRuntime().maxMemory();
@@ -189,8 +203,8 @@
return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES));
}
- public int getMaxConcurrentFlushes() {
- return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_MAX_CONCURRENT_FLUSHES);
+ public int getMaxScheduledFlushes() {
+ return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_MAX_SCHEDULED_FLUSHES);
}
public long getJobExecutionMemoryBudget() {
@@ -213,8 +227,8 @@
return accessor.getString(Option.STORAGE_IO_SCHEDULER);
}
- public int getMaxRunningFlushes(int numPartitions) {
- int value = accessor.getInt(Option.STORAGE_MAX_RUNNING_FLUSHES_PER_PARTITION);
+ public int geMaxConcurrentFlushes(int numPartitions) {
+ int value = accessor.getInt(Option.STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION);
return value != 0 ? value * numPartitions : Integer.MAX_VALUE;
}
@@ -223,11 +237,19 @@
return value != 0 ? value * numPartitions : Integer.MAX_VALUE;
}
- public int getMaxRunningMerges(int numPartitions) {
- int value = accessor.getInt(Option.STORAGE_MAX_RUNNING_MERGES_PER_PARTITION);
+ public int getMaxConcurrentMerges(int numPartitions) {
+ int value = accessor.getInt(Option.STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION);
return value != 0 ? value * numPartitions : Integer.MAX_VALUE;
}
+ public boolean isStorageGlobalCleanup() {
+ return accessor.getBoolean(Option.STORAGE_GLOBAL_CLEANUP);
+ }
+
+ public int getStorageGlobalCleanupTimeout() {
+ return accessor.getInt(Option.STORAGE_GLOBAL_CLEANUP_TIMEOUT);
+ }
+
protected int getMetadataDatasets() {
return MetadataIndexImmutableProperties.METADATA_DATASETS_COUNT;
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
index 050a493..d67e9a6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN;
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
@@ -41,7 +42,7 @@
TXN_DATASET_CHECKPOINT_INTERVAL(
POSITIVE_INTEGER,
(int) TimeUnit.MINUTES.toSeconds(60),
- "The interval (in seconds) after which a dataset is considered idle and persisted to disk"),
+ "The interval (in seconds) after which a " + dataset() + " is considered idle and persisted to disk"),
TXN_LOG_BUFFER_NUMPAGES(POSITIVE_INTEGER, 8, "The number of pages in the transaction log tail"),
TXN_LOG_BUFFER_PAGESIZE(
INTEGER_BYTE_UNIT,
@@ -65,7 +66,7 @@
TXN_LOCK_ESCALATIONTHRESHOLD(
NONNEGATIVE_INTEGER,
1000,
- "The maximum number of entity locks to obtain before upgrading to a dataset lock"),
+ "The maximum number of entity locks to obtain before upgrading to a " + dataset() + " lock"),
TXN_LOCK_SHRINKTIMER(
POSITIVE_INTEGER,
5000,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java
index 9dffa94..7b5e4a0 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java
@@ -28,7 +28,9 @@
private WarningUtil() {
}
- /** Merges the warnings from the collection argument into the warning collector argument. */
+ /**
+ * Merges the warnings from the collection argument into the warning collector argument.
+ */
public static void mergeWarnings(Collection<Warning> warnings, IWarningCollector warningsCollector) {
for (Warning warning : warnings) {
if (warningsCollector.shouldWarn()) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java
index b8124a2..f943dea 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java
@@ -36,11 +36,10 @@
* <p>
* Each dataverse name can be encoded into a single string (called a canonical form) by
* {@link #getCanonicalForm()} and decoded back from it with {@link #createFromCanonicalForm(String)}.
- * The canonical form encoding concatenates name parts together with {@link #CANONICAL_FORM_SEPARATOR_CHAR '.'}
- * character. The {@link #CANONICAL_FORM_ESCAPE_CHAR '@'} character is used to escape
- * {@link #CANONICAL_FORM_SEPARATOR_CHAR '.'} and itself in each name part prior to concatenation.
+ * The canonical form encoding concatenates name parts together with {@link #CANONICAL_FORM_SEPARATOR_CHAR '/'}
+ * character.
* <p>
- * E.g. the canonical form for a dataverse name {@code ["a", "b", "c"]} is {@code "a.b.c"}
+ * E.g. the canonical form for a dataverse name {@code ["a", "b", "c"]} is {@code "a/b/c"}
* <p>
* {@link #toString()} returns a display form which is suitable for error messages,
* and is a valid SQL++ multi-part identifier parsable by {@code IParser#parseMultipartIdentifier()}
@@ -59,11 +58,9 @@
*/
public final class DataverseName implements Serializable, Comparable<DataverseName> {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
- public static final char CANONICAL_FORM_SEPARATOR_CHAR = '.';
-
- private static final char CANONICAL_FORM_ESCAPE_CHAR = '@';
+ public static final char CANONICAL_FORM_SEPARATOR_CHAR = '/';
public static final char DISPLAY_FORM_SEPARATOR_CHAR = '.';
@@ -72,7 +69,7 @@
private static final char DISPLAY_FORM_ESCAPE_CHAR = '\\';
private static final char[] CANONICAL_FORM_SEPARATOR_AND_ESCAPE_CHARS =
- new char[] { CANONICAL_FORM_SEPARATOR_CHAR, CANONICAL_FORM_ESCAPE_CHAR };
+ new char[] { CANONICAL_FORM_SEPARATOR_CHAR };
private final boolean isMultiPart;
@@ -282,13 +279,7 @@
}
private static void encodePartIntoCanonicalForm(String part, StringBuilder out) {
- for (int i = 0, ln = part.length(); i < ln; i++) {
- char c = part.charAt(i);
- if (c == CANONICAL_FORM_SEPARATOR_CHAR || c == CANONICAL_FORM_ESCAPE_CHAR) {
- out.append(CANONICAL_FORM_ESCAPE_CHAR);
- }
- out.append(c);
- }
+ out.append(part);
}
private static <T> void decodeCanonicalForm(String canonicalForm, BiConsumer<CharSequence, T> partConsumer,
@@ -297,18 +288,11 @@
StringBuilder sb = new StringBuilder(ln);
for (int i = 0; i < ln; i++) {
char c = canonicalForm.charAt(i);
- switch (c) {
- case CANONICAL_FORM_SEPARATOR_CHAR:
- partConsumer.accept(sb, partConsumerArg);
- sb.setLength(0);
- break;
- case CANONICAL_FORM_ESCAPE_CHAR:
- i++;
- c = canonicalForm.charAt(i);
- // fall through to 'default'
- default:
- sb.append(c);
- break;
+ if (c == CANONICAL_FORM_SEPARATOR_CHAR) {
+ partConsumer.accept(sb, partConsumerArg);
+ sb.setLength(0);
+ } else {
+ sb.append(c);
}
}
if (sb.length() > 0) {
@@ -318,41 +302,11 @@
// optimization for a single part name
private static String decodeSinglePartNameFromCanonicalForm(String canonicalForm) {
- if (canonicalForm.indexOf(CANONICAL_FORM_ESCAPE_CHAR) < 0) {
- // no escaping was done
- return canonicalForm;
- }
-
- StringBuilder singlePart = new StringBuilder(canonicalForm.length());
- for (int i = 0, ln = canonicalForm.length(); i < ln; i++) {
- char c = canonicalForm.charAt(i);
- switch (c) {
- case CANONICAL_FORM_SEPARATOR_CHAR:
- throw new IllegalStateException(canonicalForm); // should never happen
- case CANONICAL_FORM_ESCAPE_CHAR:
- i++;
- c = canonicalForm.charAt(i);
- // fall through to 'default'
- default:
- singlePart.append(c);
- break;
- }
- }
- return singlePart.toString();
+ return canonicalForm;
}
private static boolean isMultiPartCanonicalForm(String canonicalForm) {
- for (int i = 0, ln = canonicalForm.length(); i < ln; i++) {
- char c = canonicalForm.charAt(i);
- switch (c) {
- case CANONICAL_FORM_SEPARATOR_CHAR:
- return true;
- case CANONICAL_FORM_ESCAPE_CHAR:
- i++;
- break;
- }
- }
- return false;
+ return canonicalForm.indexOf(CANONICAL_FORM_SEPARATOR_CHAR) != -1;
}
private static void addPartToCollection(CharSequence part, Collection<? super String> out) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierMappingUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierMappingUtil.java
new file mode 100644
index 0000000..c52f27a
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierMappingUtil.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.utils;
+
+import org.apache.asterix.common.api.IIdentifierMapper;
+
+public class IdentifierMappingUtil {
+
+ private static final IIdentifierMapper DEFAULT_MAPPER = identifier -> identifier;
+
+ private static IIdentifierMapper mapper = DEFAULT_MAPPER;
+
+ private IdentifierMappingUtil() {
+ }
+
+ public static void setMapper(IIdentifierMapper mapper) {
+ IdentifierMappingUtil.mapper = mapper;
+ }
+
+ public static String map(String key) {
+ return mapper.map(key);
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierUtil.java
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
copy to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierUtil.java
index 5812379..ebdd740 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierUtil.java
@@ -17,8 +17,18 @@
* under the License.
*/
-use tpch;
+package org.apache.asterix.common.utils;
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+public class IdentifierUtil {
+
+ public static final String DATASET = "dataset";
+ public static final String DATAVERSE = "dataverse";
+
+ public static String dataset() {
+ return IdentifierMappingUtil.map(DATASET);
+ }
+
+ public static String dataverse() {
+ return IdentifierMappingUtil.map(DATAVERSE);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index de21959..7dc2c56 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -163,8 +163,8 @@
1073 = Cannot resolve alias reference for undefined identifier %1$s
1074 = Cannot resolve ambiguous alias reference for identifier %1$s
1075 = Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.
-1076 = Cannot find dataset %1$s because there is no dataverse declared, nor an alias with name %1$s!
-1077 = Cannot find dataset %1$s in dataverse %2$s nor an alias with name %1$s!
+1076 = Cannot find dataset %1$s because there is no dataverse declared, nor an alias with name %1$s
+1077 = Cannot find dataset %1$s in dataverse %2$s nor an alias with name %1$s
1078 = Unexpected operator %1$s in an OperatorExpr starting with %2$s
1079 = Compilation error: %1$s
1080 = Cannot find node group with name %1$s
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java
index 2f0dff5..75b3989 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java
@@ -51,9 +51,7 @@
private static final List<String> TEST_BUILTIN_DATAVERSE_INVALID_NAME_PARAMS = Arrays.asList(
// separator character is not allowed
- "a.b",
- // escape character is not allowed
- "c@d");
+ "a/b");
private static final List<Triple<String, String, String>> TEST_SINGLE_PART_NAME_PARAMS = Arrays.asList(
// <1-part-name, canonical-form, display-form>
@@ -63,30 +61,30 @@
// letters and digits
new Triple<>("aA09", "aA09", "aA09"),
// with canonical form escape character
- new Triple<>("a@b", "a@@b", "`a@b`"),
+ new Triple<>("a@b", "a@b", "`a@b`"),
// with canonical form separator character
- new Triple<>("a.b", "a@.b", "`a.b`"),
+ new Triple<>("a.b", "a.b", "`a.b`"),
// with canonical form escape and separator characters
- new Triple<>("a@.b", "a@@@.b", "`a@.b`"),
+ new Triple<>("a@.b", "a@.b", "`a@.b`"),
// with display form escape character
new Triple<>("a\\b", "a\\b", "`a\\\\b`"));
private static final List<Triple<List<String>, String, String>> TEST_MULTI_PART_NAME_PARAMS = Arrays.asList(
// <multi-part-name, canonical-form, display-form>
- new Triple<>(Arrays.asList("aa", "bb", "cc"), "aa.bb.cc", "aa.bb.cc"),
+ new Triple<>(Arrays.asList("aa", "bb", "cc"), "aa/bb/cc", "aa.bb.cc"),
// mixed case letters, digits
- new Triple<>(Arrays.asList("az", "AZ", "a09Z"), "az.AZ.a09Z", "az.AZ.a09Z"),
+ new Triple<>(Arrays.asList("az", "AZ", "a09Z"), "az/AZ/a09Z", "az.AZ.a09Z"),
// with canonical form escape character
- new Triple<>(Arrays.asList("a@a@", "@b@b", "@c@c"), "a@@a@@.@@b@@b.@@c@@c", "`a@a@`.`@b@b`.`@c@c`"),
+ new Triple<>(Arrays.asList("a@a@", "@b@b", "@c@c"), "a@a@/@b@b/@c@c", "`a@a@`.`@b@b`.`@c@c`"),
// with canonical form separator character
- new Triple<>(Arrays.asList("a.a.", ".b.b.", ".c.c"), "a@.a@..@.b@.b@..@.c@.c", "`a.a.`.`.b.b.`.`.c.c`"),
+ new Triple<>(Arrays.asList("a.a.", ".b.b.", ".c.c"), "a.a./.b.b./.c.c", "`a.a.`.`.b.b.`.`.c.c`"),
// with canonical form escape and separator characters
- new Triple<>(Arrays.asList("a@a.", "@b.b@", ".c@c"), "a@@a@..@@b@.b@@.@.c@@c", "`a@a.`.`@b.b@`.`.c@c`"),
+ new Triple<>(Arrays.asList("a@a.", "@b.b@", ".c@c"), "a@a./@b.b@/.c@c", "`a@a.`.`@b.b@`.`.c@c`"),
// with canonical form escape and separator characters repeated
- new Triple<>(Arrays.asList("a@@a..", "@@b..b@@", "..c@@c"), "a@@@@a@.@..@@@@b@.@.b@@@@.@.@.c@@@@c",
+ new Triple<>(Arrays.asList("a@@a..", "@@b..b@@", "..c@@c"), "a@@a../@@b..b@@/..c@@c",
"`a@@a..`.`@@b..b@@`.`..c@@c`"),
// with display form escape character
- new Triple<>(Arrays.asList("a\\b", "c\\d"), "a\\b.c\\d", "`a\\\\b`.`c\\\\d`"));
+ new Triple<>(Arrays.asList("a\\b", "c\\d"), "a\\b/c\\d", "`a\\\\b`.`c\\\\d`"));
@Test
public void testBuiltinDataverseName() throws Exception {
@@ -220,7 +218,6 @@
testRuntimeException(() -> DataverseName.createBuiltinDataverseName(null), NullPointerException.class);
testRuntimeException(() -> DataverseName.createFromCanonicalForm(null), NullPointerException.class);
testRuntimeException(() -> DataverseName.create(Collections.singletonList(null)), NullPointerException.class);
- testRuntimeException(() -> DataverseName.create(Arrays.asList(null, null)), NullPointerException.class);
// 3. IndexOutOfBoundsException
testRuntimeException(() -> DataverseName.create(Collections.emptyList(), 0, 1),
IndexOutOfBoundsException.class);
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md
index ea4c6a15..502647e 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md
@@ -71,7 +71,7 @@
If we have a typo as above in "customers" that misses the dataset name's ending "s",
we will get an identifier resolution error as follows:
- ERROR: Code: 1 "ASX1077: Cannot find dataset customer in dataverse Commerce nor an alias with name customer! (in line 2, at column 7)"
+ ERROR: Code: 1 "ASX1077: Cannot find dataset customer in dataverse Commerce nor an alias with name customer (in line 2, at column 7)"
##### Example
(Q4.4)
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
index b2b893b..1569d69 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
@@ -452,8 +452,9 @@
The `CREATE SYNONYM` statement creates a synonym for a given dataset.
-This synonym may be used used instead of the dataset name in `SELECT`, `INSERT`, `UPSERT`, `DELETE`, and `LOAD` statements.
+This synonym may be used instead of the dataset name in `SELECT`, `INSERT`, `UPSERT`, `DELETE`, and `LOAD` statements.
The target dataset does not need to exist when the synonym is created.
+A synonym may be created for another synonym.
##### Example
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index cb97526..0e6ddb2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -23,6 +23,7 @@
import java.util.function.LongSupplier;
import java.util.function.Supplier;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
@@ -95,4 +96,8 @@
default LongSupplier getLineNumber() {
return ExternalDataConstants.NO_LINES;
}
+
+ default void handleGenericEvent(ActiveManagerMessage event) {
+ throw new IllegalStateException("unexpected generic event " + event);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
index b58e604..94d9e6e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
@@ -21,6 +21,7 @@
import java.io.Closeable;
import java.io.IOException;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -57,4 +58,8 @@
feedLogManager.close();
}
}
+
+ public void handleGenericEvent(ActiveManagerMessage event) {
+ throw new IllegalStateException("unexpected generic event " + event);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 56257a8..8cec5de 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.api.IRawRecord;
@@ -280,4 +281,9 @@
.append("}");
return str.toString();
}
+
+ @Override
+ public void handleGenericEvent(ActiveManagerMessage event) {
+ recordReader.handleGenericEvent(event);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java
index 54e633a..fc9b727 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java
@@ -21,6 +21,7 @@
import java.io.Closeable;
import java.io.IOException;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.hyracks.api.comm.IFrameWriter;
@@ -69,4 +70,8 @@
public void close() throws IOException {
controller.close();
}
+
+ public void handleGenericEvent(ActiveManagerMessage event) {
+ controller.handleGenericEvent(event);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
index 898f828..b37bce7 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
@@ -52,9 +52,6 @@
// Finished reading all the files
if (nextFileIndex >= filePaths.size()) {
- if (in != null) {
- CleanupUtils.close(in, null);
- }
return false;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
index ca55b6f..2a063bf 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
@@ -30,6 +30,7 @@
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
@@ -69,29 +70,16 @@
return partitionConstraint;
}
+ protected int getPartitionsCount() {
+ return getPartitionConstraint().getLocations().length;
+ }
+
@Override
- public abstract void configure(IServiceContext ctx, Map<String, String> configuration,
- IWarningCollector warningCollector) throws AlgebricksException;
-
- /**
- * Finds the smallest workload and returns it
- *
- * @return the smallest workload
- */
- protected PartitionWorkLoadBasedOnSize getSmallestWorkLoad() {
- PartitionWorkLoadBasedOnSize smallest = partitionWorkLoadsBasedOnSize.get(0);
- for (PartitionWorkLoadBasedOnSize partition : partitionWorkLoadsBasedOnSize) {
- // If the current total size is 0, add the file directly as this is a first time partition
- if (partition.getTotalSize() == 0) {
- smallest = partition;
- break;
- }
- if (partition.getTotalSize() < smallest.getTotalSize()) {
- smallest = partition;
- }
- }
-
- return smallest;
+ public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
+ throws AlgebricksException {
+ this.configuration = configuration;
+ this.partitionConstraint =
+ ((ICcApplicationContext) ctx.getApplicationContext()).getClusterStateManager().getClusterLocations();
}
protected IncludeExcludeMatcher getIncludeExcludeMatchers() throws CompilationException {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index e3e53d5..5eb8475 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -31,50 +32,105 @@
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.util.LogRedactionUtil;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.S3Exception;
public class AwsS3InputStream extends AbstractExternalInputStream {
+ // Configuration
+ private final String bucket;
private final S3Client s3Client;
+ private static final int MAX_RETRIES = 5; // We will retry 5 times in case of internal error from AWS S3 service
public AwsS3InputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
super(configuration, filePaths);
this.s3Client = buildAwsS3Client(configuration);
+ this.bucket = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
}
@Override
protected boolean getInputStream() throws IOException {
- String bucket = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String fileName = filePaths.get(nextFileIndex);
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(filePaths.get(nextFileIndex)).build();
-
// Have a reference to the S3 stream to ensure that if GZipInputStream causes an IOException because of reading
// the header, then the S3 stream gets closed in the close method
- try {
- in = s3Client.getObject(getObjectRequest);
- } catch (NoSuchKeyException ex) {
- LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(getObjectRequest.key()) + " was not found in bucket "
- + getObjectRequest.bucket());
+ if (!doGetInputStream(getObjectRequest)) {
return false;
- } catch (SdkException ex) {
- throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
-
// Use gzip stream if needed
- String filename = filePaths.get(nextFileIndex).toLowerCase();
- if (filename.endsWith(".gz") || filename.endsWith(".gzip")) {
- in = new GZIPInputStream(s3Client.getObject(getObjectRequest), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ if (StringUtils.endsWithIgnoreCase(fileName, ".gz") || StringUtils.endsWithIgnoreCase(fileName, ".gzip")) {
+ in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
}
-
return true;
}
+ /**
+ * Get the input stream. If an error is encountered, depending on the error code, a retry might be favorable.
+ *
+ * @return true
+ */
+ private boolean doGetInputStream(GetObjectRequest request) throws RuntimeDataException {
+ int retries = 0;
+ while (retries < MAX_RETRIES) {
+ try {
+ in = s3Client.getObject(request);
+ break;
+ } catch (NoSuchKeyException ex) {
+ LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(request.key()) + " was not found in bucket "
+ + request.bucket());
+ return false;
+ } catch (S3Exception ex) {
+ if (!shouldRetry(ex.awsErrorDetails().errorCode(), retries++)) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ }
+ LOGGER.debug(() -> "S3 retryable error: " + LogRedactionUtil.userData(ex.getMessage()));
+
+ // Backoff for 1 sec for the first 2 retries, and 2 seconds from there onward
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(retries < 3 ? 1 : 2));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ } catch (SdkException ex) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ }
+ }
+ return true;
+ }
+
+ private boolean shouldRetry(String errorCode, int currentRetry) {
+ return currentRetry < MAX_RETRIES && AwsS3.isRetryableError(errorCode);
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (in != null) {
+ CleanupUtils.close(in, null);
+ }
+ if (s3Client != null) {
+ CleanupUtils.close(s3Client, null);
+ }
+ }
+
+ @Override
+ public boolean stop() {
+ try {
+ close();
+ } catch (IOException e) {
+ // Ignore
+ }
+ return false;
+ }
+
private S3Client buildAwsS3Client(Map<String, String> configuration) throws HyracksDataException {
try {
return ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 5bab888..8197524 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -18,19 +18,19 @@
*/
package org.apache.asterix.external.input.record.reader.aws;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
-
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.PriorityQueue;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
@@ -61,8 +61,7 @@
@Override
public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
throws AlgebricksException {
- this.configuration = configuration;
- ICcApplicationContext ccApplicationContext = (ICcApplicationContext) ctx.getApplicationContext();
+ super.configure(ctx, configuration, warningCollector);
// Ensure the validity of include/exclude
ExternalDataUtils.validateIncludeExclude(configuration);
@@ -70,7 +69,7 @@
// Prepare to retrieve the objects
List<S3Object> filesOnly;
- String container = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
S3Client s3Client = ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
try {
@@ -79,7 +78,7 @@
// New API is not implemented, try falling back to old API
try {
// For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode().equals("NotImplemented")) {
+ if (ex.awsErrorDetails().errorCode().equals(ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED)) {
filesOnly = oldApiListS3Objects(s3Client, container, includeExcludeMatcher);
} else {
throw ex;
@@ -101,12 +100,8 @@
warningCollector.warn(warning);
}
- // Partition constraints
- partitionConstraint = ccApplicationContext.getClusterStateManager().getClusterLocations();
- int partitionsCount = partitionConstraint.getLocations().length;
-
// Distribute work load amongst the partitions
- distributeWorkLoad(filesOnly, partitionsCount);
+ distributeWorkLoad(filesOnly, getPartitionsCount());
}
/**
@@ -223,14 +218,19 @@
* @param partitionsCount Partitions count
*/
private void distributeWorkLoad(List<S3Object> fileObjects, int partitionsCount) {
+ PriorityQueue<PartitionWorkLoadBasedOnSize> workloadQueue = new PriorityQueue<>(partitionsCount,
+ Comparator.comparingLong(PartitionWorkLoadBasedOnSize::getTotalSize));
+
// Prepare the workloads based on the number of partitions
for (int i = 0; i < partitionsCount; i++) {
- partitionWorkLoadsBasedOnSize.add(new PartitionWorkLoadBasedOnSize());
+ workloadQueue.add(new PartitionWorkLoadBasedOnSize());
}
for (S3Object object : fileObjects) {
- PartitionWorkLoadBasedOnSize smallest = getSmallestWorkLoad();
- smallest.addFilePath(object.key(), object.size());
+ PartitionWorkLoadBasedOnSize workload = workloadQueue.poll();
+ workload.addFilePath(object.key(), object.size());
+ workloadQueue.add(workload);
}
+ partitionWorkLoadsBasedOnSize.addAll(workloadQueue);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java
index 358c412..3fb3395 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.external.input.record.reader.azure;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob;
-
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -43,15 +41,17 @@
public class AzureBlobInputStream extends AbstractExternalInputStream {
private final BlobServiceClient client;
+ private final String container;
public AzureBlobInputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
super(configuration, filePaths);
this.client = buildAzureClient(configuration);
+ this.container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
}
@Override
protected boolean getInputStream() throws IOException {
- String container = configuration.get(AzureBlob.CONTAINER_NAME_FIELD_NAME);
+ String fileName = filePaths.get(nextFileIndex);
BlobContainerClient blobContainerClient;
BlobClient blob;
try {
@@ -60,9 +60,9 @@
in = blob.openInputStream();
// Use gzip stream if needed
- String filename = filePaths.get(nextFileIndex).toLowerCase();
- if (filename.endsWith(".gz") || filename.endsWith(".gzip")) {
- in = new GZIPInputStream(in = blob.openInputStream(), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ String lowerCaseFileName = fileName.toLowerCase();
+ if (lowerCaseFileName.endsWith(".gz") || lowerCaseFileName.endsWith(".gzip")) {
+ in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
}
} catch (BlobStorageException ex) {
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
index 167e22a..803694a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
@@ -18,19 +18,19 @@
*/
package org.apache.asterix.external.input.record.reader.azure;
-import static org.apache.asterix.external.util.ExternalDataConstants.*;
-
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.PriorityQueue;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
@@ -56,10 +56,9 @@
@Override
public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
throws AlgebricksException {
- this.configuration = configuration;
- ICcApplicationContext ccApplicationContext = (ICcApplicationContext) ctx.getApplicationContext();
+ super.configure(ctx, configuration, warningCollector);
- String container = configuration.get(AzureBlob.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
List<BlobItem> filesOnly = new ArrayList<>();
@@ -87,12 +86,8 @@
warningCollector.warn(warning);
}
- // Partition constraints
- partitionConstraint = ccApplicationContext.getClusterStateManager().getClusterLocations();
- int partitionsCount = partitionConstraint.getLocations().length;
-
// Distribute work load amongst the partitions
- distributeWorkLoad(filesOnly, partitionsCount);
+ distributeWorkLoad(filesOnly, getPartitionsCount());
} catch (Exception ex) {
throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
@@ -138,14 +133,19 @@
* @param partitionsCount Partitions count
*/
private void distributeWorkLoad(List<BlobItem> items, int partitionsCount) {
+ PriorityQueue<PartitionWorkLoadBasedOnSize> workloadQueue = new PriorityQueue<>(partitionsCount,
+ Comparator.comparingLong(PartitionWorkLoadBasedOnSize::getTotalSize));
+
// Prepare the workloads based on the number of partitions
for (int i = 0; i < partitionsCount; i++) {
- partitionWorkLoadsBasedOnSize.add(new PartitionWorkLoadBasedOnSize());
+ workloadQueue.add(new PartitionWorkLoadBasedOnSize());
}
- for (BlobItem item : items) {
- PartitionWorkLoadBasedOnSize smallest = getSmallestWorkLoad();
- smallest.addFilePath(item.getName(), item.getProperties().getContentLength());
+ for (BlobItem object : items) {
+ PartitionWorkLoadBasedOnSize workload = workloadQueue.poll();
+ workload.addFilePath(object.getName(), object.getProperties().getContentLength());
+ workloadQueue.add(workload);
}
+ partitionWorkLoadsBasedOnSize.addAll(workloadQueue);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
index cb16de5..1e003a1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
@@ -33,7 +33,6 @@
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.input.stream.AsterixInputStreamReader;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -51,9 +50,10 @@
private Supplier<String> previousDataSourceName = EMPTY_STRING;
public void configure(AsterixInputStream inputStream, Map<String, String> config) {
- this.reader = new AsterixInputStreamReader(inputStream);
+ int bufferSize = ExternalDataUtils.getOrDefaultBufferSize(config);
+ this.reader = new AsterixInputStreamReader(inputStream, bufferSize);
record = new CharArrayRecord();
- inputBuffer = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
+ inputBuffer = new char[bufferSize];
if (!ExternalDataUtils.isTrue(config, KEY_REDACT_WARNINGS)) {
this.dataSourceName = reader::getStreamName;
this.previousDataSourceName = reader::getPreviousStreamName;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
index d2499f8..81da0b0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
@@ -27,25 +27,24 @@
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class AsterixInputStreamReader extends Reader {
private final AsterixInputStream in;
private final CharsetDecoder decoder;
- private final byte[] bytes;
+ private byte[] bytes;
protected final ByteBuffer byteBuffer;
protected final CharBuffer charBuffer;
protected boolean done = false;
protected boolean remaining = false;
- public AsterixInputStreamReader(AsterixInputStream in) {
+ public AsterixInputStreamReader(AsterixInputStream in, int bufferSize) {
this.in = in;
this.decoder = StandardCharsets.UTF_8.newDecoder();
- bytes = new byte[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
- byteBuffer = ByteBuffer.wrap(bytes);
- charBuffer = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ this.bytes = new byte[bufferSize];
+ this.byteBuffer = ByteBuffer.wrap(bytes);
+ this.charBuffer = CharBuffer.allocate(bufferSize);
this.byteBuffer.flip();
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/StandardUTF8ToModifiedUTF8DataOutput.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/StandardUTF8ToModifiedUTF8DataOutput.java
index be0d286..adeb80e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/StandardUTF8ToModifiedUTF8DataOutput.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/StandardUTF8ToModifiedUTF8DataOutput.java
@@ -141,7 +141,7 @@
private final ByteArrayAccessibleInputStream inByte;
public ResettableUTF8InputStreamReader(ByteArrayAccessibleInputStream inByte) {
- super(new BasicInputStream(inByte));
+ super(new BasicInputStream(inByte), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
this.inByte = inByte;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
index e2229ee..457b86a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
@@ -27,6 +27,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -65,18 +66,21 @@
private IPCSystem ipcSys;
private String sitePkgs;
private List<String> pythonArgs;
+ private Map<String, String> pythonEnv;
private TaskAttemptId task;
private IWarningCollector warningCollector;
private SourceLocation sourceLoc;
public PythonLibraryEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId, ILibraryManager libMgr,
- File pythonHome, String sitePkgs, List<String> pythonArgs, ExternalFunctionResultRouter router,
- IPCSystem ipcSys, TaskAttemptId task, IWarningCollector warningCollector, SourceLocation sourceLoc) {
+ File pythonHome, String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv,
+ ExternalFunctionResultRouter router, IPCSystem ipcSys, TaskAttemptId task,
+ IWarningCollector warningCollector, SourceLocation sourceLoc) {
super(jobId, evaluatorId);
this.libMgr = libMgr;
this.pythonHome = pythonHome;
this.sitePkgs = sitePkgs;
this.pythonArgs = pythonArgs;
+ this.pythonEnv = pythonEnv;
this.router = router;
this.task = task;
this.ipcSys = ipcSys;
@@ -98,8 +102,8 @@
args.add(InetAddress.getLoopbackAddress().getHostAddress());
args.add(Integer.toString(port));
args.add(sitePkgs);
-
ProcessBuilder pb = new ProcessBuilder(args.toArray(new String[0]));
+ pb.environment().putAll(pythonEnv);
pb.directory(new File(wd));
p = pb.start();
proto = new PythonIPCProto(p.getOutputStream(), router, p);
@@ -199,14 +203,15 @@
public static PythonLibraryEvaluator getInstance(IExternalFunctionInfo finfo, ILibraryManager libMgr,
ExternalFunctionResultRouter router, IPCSystem ipcSys, File pythonHome, IHyracksTaskContext ctx,
- String sitePkgs, List<String> pythonArgs, IWarningCollector warningCollector, SourceLocation sourceLoc)
- throws IOException, AsterixException {
+ String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv, IWarningCollector warningCollector,
+ SourceLocation sourceLoc) throws IOException, AsterixException {
PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
finfo.getLibraryName(), Thread.currentThread());
PythonLibraryEvaluator evaluator = (PythonLibraryEvaluator) ctx.getStateObject(evaluatorId);
if (evaluator == null) {
evaluator = new PythonLibraryEvaluator(ctx.getJobletContext().getJobId(), evaluatorId, libMgr, pythonHome,
- sitePkgs, pythonArgs, router, ipcSys, ctx.getTaskAttemptId(), warningCollector, sourceLoc);
+ sitePkgs, pythonArgs, pythonEnv, router, ipcSys, ctx.getTaskAttemptId(), warningCollector,
+ sourceLoc);
ctx.getJobletContext().registerDeallocatable(evaluator);
evaluator.initialize();
ctx.setStateObject(evaluator);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
index 86d51de..06c9bc9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
@@ -23,8 +23,9 @@
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -46,6 +47,7 @@
private final ExternalFunctionResultRouter router;
private final String sitePackagesPath;
private final List<String> pythonArgs;
+ private final Map<String, String> pythonEnv;
public PythonLibraryEvaluatorFactory(IHyracksTaskContext ctx) throws AsterixException {
this.ctx = ctx;
@@ -67,17 +69,41 @@
+ NCConfig.Option.PYTHON_CMD_AUTOLOCATE.ini() + " is false");
}
}
+ pythonEnv = new HashMap<>();
+ String[] envRaw = appCfg.getStringArray((NCConfig.Option.PYTHON_ENV));
+ if (envRaw != null) {
+ for (String rawEnvArg : envRaw) {
+ //TODO: i think equals is shared among all unixes and windows. but it needs verification
+ if (rawEnvArg.length() < 1) {
+ continue;
+ }
+ String[] rawArgSplit = rawEnvArg.split("(?<!\\\\)=", 2);
+ if (rawArgSplit.length < 2) {
+ throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ "Invalid environment variable format detected.");
+ }
+ pythonEnv.put(rawArgSplit[0], rawArgSplit[1]);
+ }
+ }
pythonPath = new File(pythonPathCmd);
List<String> sitePkgs = new ArrayList<>();
sitePkgs.add(SITE_PACKAGES);
String[] addlSitePackages = appCfg.getStringArray((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES));
- sitePkgs.addAll(Arrays.asList(addlSitePackages));
+ for (String sitePkg : addlSitePackages) {
+ if (sitePkg.length() > 0) {
+ sitePkgs.add(sitePkg);
+ }
+ }
if (appCfg.getBoolean(NCConfig.Option.PYTHON_USE_BUNDLED_MSGPACK)) {
sitePkgs.add("ipc" + File.separator + SITE_PACKAGES + File.separator);
}
String[] pythonArgsRaw = appCfg.getStringArray(NCConfig.Option.PYTHON_ARGS);
if (pythonArgsRaw != null) {
- pythonArgs.addAll(Arrays.asList(pythonArgsRaw));
+ for (String arg : pythonArgsRaw) {
+ if (arg.length() > 0) {
+ pythonArgs.add(arg);
+ }
+ }
}
StringBuilder sitePackagesPathBuilder = new StringBuilder();
for (int i = 0; i < sitePkgs.size() - 1; i++) {
@@ -91,6 +117,6 @@
public PythonLibraryEvaluator getEvaluator(IExternalFunctionInfo fnInfo, SourceLocation sourceLoc)
throws IOException, AsterixException {
return PythonLibraryEvaluator.getInstance(fnInfo, libraryManager, router, ipcSys, pythonPath, ctx,
- sitePackagesPath, pythonArgs, ctx.getWarningCollector(), sourceLoc);
+ sitePackagesPath, pythonArgs, pythonEnv, ctx.getWarningCollector(), sourceLoc);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index fd5b269..a0cf387 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -120,6 +120,7 @@
public static final String KEY_NULL_STR = "null";
public static final String KEY_REDACT_WARNINGS = "redact-warnings";
public static final String KEY_REQUESTED_FIELDS = "requested-fields";
+ public static final String KEY_EXTERNAL_SCAN_BUFFER_SIZE = "external-scan-buffer-size";
/**
* Keys for adapter name
@@ -288,6 +289,7 @@
public static final String INVALID_VAL = "invalid value";
public static final String DEFINITION_FIELD_NAME = "definition";
+ public static final String CONTAINER_NAME_FIELD_NAME = "container";
public static class AwsS3 {
private AwsS3() {
@@ -298,8 +300,16 @@
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
- public static final String CONTAINER_NAME_FIELD_NAME = "container";
public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
+
+ // AWS S3 specific error codes
+ public static final String ERROR_INTERNAL_ERROR = "InternalError";
+ public static final String ERROR_SLOW_DOWN = "SlowDown";
+ public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
+
+ public static boolean isRetryableError(String errorCode) {
+ return errorCode.equals(ERROR_INTERNAL_ERROR) || errorCode.equals(ERROR_SLOW_DOWN);
+ }
}
public static class AzureBlob {
@@ -307,8 +317,6 @@
throw new AssertionError("do not instantiate");
}
- public static final String CONTAINER_NAME_FIELD_NAME = "container";
- public static final String DEFINITION_FIELD_NAME = "definition";
public static final String CONNECTION_STRING_FIELD_NAME = "connectionString";
public static final String ACCOUNT_NAME_FIELD_NAME = "accountName";
public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 3ac1116..4beb932 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
@@ -31,6 +32,7 @@
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START;
@@ -113,6 +115,11 @@
private ExternalDataUtils() {
}
+ public static int getOrDefaultBufferSize(Map<String, String> configuration) {
+ String bufferSize = configuration.get(KEY_EXTERNAL_SCAN_BUFFER_SIZE);
+ return bufferSize != null ? Integer.parseInt(bufferSize) : ExternalDataConstants.DEFAULT_BUFFER_SIZE;
+ }
+
// Get a delimiter from the given configuration
public static char validateGetDelimiter(Map<String, String> configuration) throws HyracksDataException {
return validateCharOrDefault(configuration, KEY_DELIMITER, ExternalDataConstants.DEFAULT_DELIMITER.charAt(0));
@@ -776,7 +783,7 @@
S3Client s3Client = buildAwsS3Client(configuration);;
S3Response response;
boolean useOldApi = false;
- String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
String prefix = getPrefix(configuration);
try {
@@ -785,7 +792,7 @@
// Method not implemented, try falling back to old API
try {
// For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode().equals("NotImplemented")) {
+ if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
useOldApi = true;
response = isBucketEmpty(s3Client, container, prefix, true);
} else {
@@ -943,7 +950,7 @@
// Check if the bucket is present
BlobServiceClient blobServiceClient;
try {
- String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
blobServiceClient = buildAzureClient(configuration);
BlobContainerClient blobContainer = blobServiceClient.getBlobContainerClient(container);
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
new file mode 100644
index 0000000..90ea04b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.awss3;
+
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_INTERNAL_ERROR;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_SLOW_DOWN;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.input.record.reader.aws.AwsS3InputStream;
+import org.apache.asterix.external.input.record.reader.aws.AwsS3InputStreamFactory;
+import org.apache.hyracks.api.exceptions.IFormattedException;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+public class AwsS3Test {
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testWorkloadDistribution() throws Exception {
+ AwsS3InputStreamFactory factory = new AwsS3InputStreamFactory();
+
+ List<S3Object> s3Objects = new ArrayList<>();
+ final int partitionsCount = 3;
+
+ // Create S3 objects, 9 objects, on 3 partitions, they should be 600 total size on each partition
+ S3Object.Builder builder = S3Object.builder();
+ s3Objects.add(builder.key("1.json").size(100L).build());
+ s3Objects.add(builder.key("2.json").size(100L).build());
+ s3Objects.add(builder.key("3.json").size(100L).build());
+ s3Objects.add(builder.key("4.json").size(200L).build());
+ s3Objects.add(builder.key("5.json").size(200L).build());
+ s3Objects.add(builder.key("6.json").size(200L).build());
+ s3Objects.add(builder.key("7.json").size(300L).build());
+ s3Objects.add(builder.key("8.json").size(300L).build());
+ s3Objects.add(builder.key("9.json").size(300L).build());
+
+ // invoke the distributeWorkLoad method
+ Method distributeWorkloadMethod =
+ AwsS3InputStreamFactory.class.getDeclaredMethod("distributeWorkLoad", List.class, int.class);
+ distributeWorkloadMethod.setAccessible(true);
+ distributeWorkloadMethod.invoke(factory, s3Objects, partitionsCount);
+
+ // get the partitionWorkLoadsBasedOnSize field and verify the result
+ Field distributeWorkloadField =
+ AwsS3InputStreamFactory.class.getSuperclass().getDeclaredField("partitionWorkLoadsBasedOnSize");
+ distributeWorkloadField.setAccessible(true);
+ List<AbstractExternalInputStreamFactory.PartitionWorkLoadBasedOnSize> workloads =
+ (List<AbstractExternalInputStreamFactory.PartitionWorkLoadBasedOnSize>) distributeWorkloadField
+ .get(factory);
+
+ for (AbstractExternalInputStreamFactory.PartitionWorkLoadBasedOnSize workload : workloads) {
+ Assert.assertEquals(workload.getTotalSize(), 600);
+ }
+ }
+
+ @Test
+ public void s3InternalError() throws Exception {
+ // S3Client mock
+ S3Client s3ClientMock = Mockito.mock(S3Client.class);
+
+ // Prepare S3Exception with internal error code
+ AwsErrorDetails errorDetails = AwsErrorDetails.builder().errorCode(ERROR_INTERNAL_ERROR)
+ .errorMessage("Internal Error from AWS").build();
+ S3Exception internalErrorEx = (S3Exception) S3Exception.builder().awsErrorDetails(errorDetails).build();
+ Mockito.when(s3ClientMock.getObject(GetObjectRequest.builder().build())).thenThrow(internalErrorEx);
+
+ // Set S3Client mock
+ AwsS3InputStream inputStreamMock = Mockito.mock(AwsS3InputStream.class);
+ Field s3ClientField = AwsS3InputStream.class.getDeclaredField("s3Client");
+ s3ClientField.setAccessible(true);
+ s3ClientField.set(inputStreamMock, s3ClientMock);
+
+ // doGetInputStream method
+ Method doGetInputStreamMethod =
+ AwsS3InputStream.class.getDeclaredMethod("doGetInputStream", GetObjectRequest.class);
+ doGetInputStreamMethod.setAccessible(true);
+
+ try {
+ doGetInputStreamMethod.invoke(inputStreamMock, GetObjectRequest.builder().build());
+ } catch (Exception ex) {
+ Assert.assertTrue("Not internal error", ex.getCause() instanceof IFormattedException
+ && ex.getCause().toString().contains("ASX1108: External source error. Internal Error from AWS"));
+ }
+ }
+
+ @Test
+ public void s3SlowDown() throws Exception {
+ // S3Client mock
+ S3Client s3ClientMock = Mockito.mock(S3Client.class);
+
+ // Prepare S3Exception with slow down error code
+ AwsErrorDetails errorDetails =
+ AwsErrorDetails.builder().errorCode(ERROR_SLOW_DOWN).errorMessage("SlowDown Error from AWS").build();
+ S3Exception slowDownEx = (S3Exception) S3Exception.builder().awsErrorDetails(errorDetails).build();
+ Mockito.when(s3ClientMock.getObject(GetObjectRequest.builder().build())).thenThrow(slowDownEx);
+
+ // Set S3Client mock
+ AwsS3InputStream inputStreamMock = Mockito.mock(AwsS3InputStream.class);
+
+ // Set S3Client
+ Field s3ClientField = AwsS3InputStream.class.getDeclaredField("s3Client");
+ s3ClientField.setAccessible(true);
+ s3ClientField.set(inputStreamMock, s3ClientMock);
+
+ // doGetInputStream method
+ Method doGetInputStreamMethod =
+ AwsS3InputStream.class.getDeclaredMethod("doGetInputStream", GetObjectRequest.class);
+ doGetInputStreamMethod.setAccessible(true);
+
+ try {
+ doGetInputStreamMethod.invoke(inputStreamMock, GetObjectRequest.builder().build());
+ } catch (Exception ex) {
+ Assert.assertTrue("Not SlowDown error", ex.getCause() instanceof IFormattedException
+ && ex.getCause().toString().contains("ASX1108: External source error. SlowDown Error from AWS"));
+ }
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 09709d08..c769960 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -20,6 +20,7 @@
package org.apache.asterix.metadata;
import static org.apache.asterix.common.exceptions.ErrorCode.FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.rmi.RemoteException;
import java.util.ArrayList;
@@ -847,7 +848,7 @@
}
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "node group",
- nodeGroupName, "datasets",
+ nodeGroupName, dataset() + "(s)",
datasets.stream().map(DatasetUtil::getFullyQualifiedDisplayName).collect(Collectors.joining(", ")));
}
try {
@@ -1128,7 +1129,7 @@
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, "type",
TypeUtil.getFullyQualifiedDisplayName(set.getItemTypeDataverseName(), set.getItemTypeName()),
- "dataset", DatasetUtil.getFullyQualifiedDisplayName(set));
+ dataset(), DatasetUtil.getFullyQualifiedDisplayName(set));
}
if (set.getMetaItemTypeDataverseName() != null
&& set.getMetaItemTypeDataverseName().equals(dataverseName)) {
@@ -1136,7 +1137,7 @@
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, "type",
TypeUtil.getFullyQualifiedDisplayName(set.getMetaItemTypeDataverseName(),
set.getMetaItemTypeName()),
- "dataset", DatasetUtil.getFullyQualifiedDisplayName(set));
+ dataset(), DatasetUtil.getFullyQualifiedDisplayName(set));
}
}
@@ -1156,9 +1157,8 @@
Function.FunctionDependencyKind functionDependencyKind = functionDependencyKinds[i];
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS,
- functionDependencyKind.toString().toLowerCase(),
- functionDependencyKind.getDependencyDisplayName(dependency), "function",
- function.getSignature());
+ functionDependencyKind, functionDependencyKind.getDependencyDisplayName(dependency),
+ "function", function.getSignature());
}
}
}
@@ -1321,7 +1321,7 @@
if (set.getItemTypeName().equals(datatypeName) && set.getItemTypeDataverseName().equals(dataverseName)) {
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "type",
- TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName), "dataset",
+ TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName), dataset(),
DatasetUtil.getFullyQualifiedDisplayName(set));
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 29a66b9..17f91b4 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.declared;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
+
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -44,6 +46,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
@@ -109,7 +112,10 @@
externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
+ PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
+ int externalScanBufferSize = physicalOptimizationConfig.getExternalScanBufferSize();
Map<String, String> properties = addProjectionInfo(projectionInfo, edd.getProperties());
+ properties.put(KEY_EXTERNAL_SCAN_BUFFER_SIZE, String.valueOf(externalScanBufferSize));
ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
index dd8e44c..1376008 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.metadata.declared;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+
import java.util.ArrayList;
import java.util.List;
@@ -78,7 +81,7 @@
return null;
}
if (dataverseName == null) {
- throw new AlgebricksException("Cannot declare output-record-type with no dataverse!");
+ throw new AlgebricksException("Cannot declare output-record-type with no " + dataverse());
}
IAType type = findType(mdTxnCtx, dataverseName, outputRecordType);
if (!(type instanceof ARecordType)) {
@@ -109,7 +112,8 @@
String datasetName) throws AlgebricksException {
Dataset dataset = findDataset(mdTxnCtx, dataverseName, datasetName);
if (dataset == null) {
- throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
+ throw new AlgebricksException(
+ "Unknown " + dataset() + " " + datasetName + " in " + dataverse() + " " + dataverseName);
}
return dataset;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 9241458..497b15f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.declared;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import static org.apache.asterix.metadata.utils.MetadataConstants.METADATA_OBJECT_NAME_INVALID_CHARS;
import java.io.File;
@@ -618,8 +620,8 @@
Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), indexName);
if (secondaryIndex == null) {
- throw new AlgebricksException(
- "Code generation error: no index " + indexName + " for dataset " + dataset.getDatasetName());
+ throw new AlgebricksException("Code generation error: no index " + indexName + " for " + dataset() + " "
+ + dataset.getDatasetName());
}
Index.ValueIndexDetails secondaryIndexDetails = (Index.ValueIndexDetails) secondaryIndex.getIndexDetails();
RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
@@ -985,8 +987,8 @@
String datasetName = dataSource.getId().getDatasourceName();
Dataset dataset = findDataset(dataSource.getId().getDataverseName(), datasetName);
if (dataset == null) {
- throw new AlgebricksException(
- "Unknown dataset " + datasetName + " in dataverse " + dataSource.getId().getDataverseName());
+ throw new AlgebricksException("Unknown " + dataset() + " " + datasetName + " in " + dataverse() + " "
+ + dataSource.getId().getDataverseName());
}
int numKeys = primaryKeys.size();
int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1;
@@ -1035,7 +1037,7 @@
JobSpecification jobSpec, IAType itemType, ITypedAdapterFactory adapterFactory,
ITupleFilterFactory tupleFilterFactory, long outputLimit) throws AlgebricksException {
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
- throw new AlgebricksException("Can only scan datasets of records.");
+ throw new AlgebricksException("Can only scan " + dataset() + "s of records.");
}
ISerializerDeserializer<?> payloadSerde =
@@ -1090,7 +1092,7 @@
keyType = IndexingConstants.getFieldType(j);
break;
default:
- throw new AlgebricksException("Unknown Dataset Type");
+ throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE, dsType.toString());
}
} catch (AsterixException e) {
throw new AlgebricksException(e);
@@ -1531,7 +1533,8 @@
// Sanity checks.
if (primaryKeys.size() > 1) {
- throw new AlgebricksException("Cannot create inverted index on dataset with composite primary key.");
+ throw new AlgebricksException(
+ "Cannot create inverted index on " + dataset() + "s with composite primary key.");
}
// The size of secondaryKeys can be two if it receives input from its
// TokenizeOperator- [token, number of token]
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
index 3cd4346..a9fa024 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.entities;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -197,5 +199,10 @@
public String getDependencyDisplayName(Triple<DataverseName, String, String> dependency) {
return dependencyDisplayNameAccessor.apply(dependency);
}
+
+ @Override
+ public String toString() {
+ return this == DATASET ? dataset() : name().toLowerCase();
+ }
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index d5e941f..92390a7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.feeds;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.rmi.RemoteException;
import java.util.Map;
@@ -53,6 +55,7 @@
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -73,12 +76,12 @@
String datasetName) throws AlgebricksException {
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
- throw new CompilationException("Unknown target dataset :" + datasetName);
+ throw new CompilationException("Unknown target " + dataset() + " :" + datasetName);
}
if (!dataset.getDatasetType().equals(DatasetType.INTERNAL)) {
- throw new CompilationException("Statement not applicable. Dataset " + datasetName
- + " is not of required type " + DatasetType.INTERNAL);
+ throw new CompilationException("Statement not applicable. " + StringUtils.capitalize(dataset()) + " "
+ + datasetName + " is not of required type " + DatasetType.INTERNAL);
}
return dataset;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4bcc5f0..4ef349b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.metadata.utils;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+
import java.io.DataOutput;
import java.util.ArrayList;
import java.util.Collections;
@@ -366,7 +369,8 @@
DataverseName dataverseName = dataverse.getDataverseName();
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
- throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
+ throw new AsterixException(
+ "Could not find " + dataset() + " " + datasetName + " in " + dataverse() + " " + dataverseName);
}
JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
index b6510b9..d0b7c59 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.utils;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.List;
import java.util.Map;
@@ -77,7 +79,8 @@
index.getIndexType().name(), dataset.getDatasetType());
}
if (numPrimaryKeys > 1) {
- throw new AsterixException("Cannot create inverted index on dataset with composite primary key.");
+ throw new AsterixException(
+ "Cannot create inverted index on " + dataset() + "s with composite primary key.");
}
if (numSecondaryKeys > 1) {
throw new AsterixException("Cannot create composite inverted index on multiple fields.");
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index 63573a1..aac01f9 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1657,7 +1657,7 @@
addFunction(CREATE_RECTANGLE, ARectangleTypeComputer.INSTANCE, true);
addFunction(CREATE_UUID, AUUIDTypeComputer.INSTANCE, false);
addFunction(UUID, AUUIDTypeComputer.INSTANCE, false);
- addFunction(CREATE_QUERY_UID, ABinaryTypeComputer.INSTANCE, false);
+ addPrivateFunction(CREATE_QUERY_UID, ABinaryTypeComputer.INSTANCE, false);
addFunction(UUID_CONSTRUCTOR, AUUIDTypeComputer.INSTANCE, true);
addFunction(RANDOM, ADoubleTypeComputer.INSTANCE, false);
addFunction(RANDOM_WITH_SEED, NumericUnaryTypeComputer.INSTANCE_DOUBLE, false);
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index d572ba2..97dd049 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -34,9 +34,13 @@
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NetworkingUtil {
+ private static final Logger LOGGER = LogManager.getLogger();
+
private NetworkingUtil() {
throw new AssertionError("This util class should not be initialized.");
}
@@ -72,9 +76,15 @@
long fileSize = fileChannel.size();
long count = fileSize;
long numTransferred = 0;
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ try {
+ while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
+ pos += numTransferred;
+ count -= numTransferred;
+ }
+ } catch (Exception e) {
+ LOGGER.info("failed to download file; file size {}, pos {}, count {}, numTransferred {}", fileSize, pos,
+ count, numTransferred);
+ throw e;
}
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 35dbd49..5f7bee6 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -63,6 +63,7 @@
@Override
public void perform(INcApplicationContext appCtx, IReplicationWorker worker) {
try {
+ LOGGER.info("attempting to replicate {}", this);
final IIOManager ioManager = appCtx.getIoManager();
// resolve path
final FileReference localPath = ioManager.resolve(file);
@@ -127,4 +128,10 @@
final boolean isMetadata = input.readBoolean();
return new ReplicateFileTask(s, i, isMetadata);
}
+
+ @Override
+ public String toString() {
+ return "ReplicateFileTask{" + "file='" + file + '\'' + ", size=" + size + ", indexMetadata=" + indexMetadata
+ + '}';
+ }
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
index f0a4229..7bb2858 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
@@ -32,9 +32,12 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class FileSynchronizer {
+ private static final Logger LOGGER = LogManager.getLogger();
private final INcApplicationContext appCtx;
private final PartitionReplica replica;
@@ -53,6 +56,7 @@
final ISocketChannel channel = replica.getChannel();
final FileReference filePath = ioManager.resolve(file);
ReplicateFileTask task = new ReplicateFileTask(file, filePath.getFile().length(), metadata);
+ LOGGER.info("attempting to replicate {} to replica {}", task, replica);
ReplicationProtocol.sendTo(replica, task);
// send the file itself
try (RandomAccessFile fromFile = new RandomAccessFile(filePath.getFile(), "r");
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java
index adb8874..86d0596 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java
@@ -19,15 +19,14 @@
package org.apache.asterix.runtime.evaluators.functions;
import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.runningaggregates.base.AbstractRunningAggregateFunctionDynamicDescriptor;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
@@ -35,31 +34,26 @@
/**
* Create global unique id within a query.
*/
-public class CreateQueryUIDDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+public class CreateQueryUIDDescriptor extends AbstractRunningAggregateFunctionDynamicDescriptor {
private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new CreateQueryUIDDescriptor();
- }
- };
+ public static final IFunctionDescriptorFactory FACTORY = CreateQueryUIDDescriptor::new;
@Override
- public IScalarEvaluatorFactory createEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
+ public IRunningAggregateEvaluatorFactory createRunningAggregateEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ return new IRunningAggregateEvaluatorFactory() {
private static final long serialVersionUID = 1L;
private static final int BINARY_LENGTH = 14;
private static final int PAYLOAD_START = 2;
@Override
- public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
+ public IRunningAggregateEvaluator createRunningAggregateEvaluator(IEvaluatorContext ctx) {
// Format: |TypeTag | PayloadLength | Payload |
// TypeTag: 1 byte
// PayloadLength: 1 byte
// Payload: 12 bytes: |partition-id (4 bytes) | local-id (8 bytes) |
- byte[] uidBytes = new byte[BINARY_LENGTH];
+ final byte[] uidBytes = new byte[BINARY_LENGTH];
// Writes the type tag.
uidBytes[0] = ATypeTag.SERIALIZED_BINARY_TYPE_TAG;
// Writes the payload size.
@@ -68,10 +62,13 @@
IntegerPointable.setInteger(uidBytes, PAYLOAD_START,
ctx.getTaskContext().getTaskAttemptId().getTaskId().getPartition());
- return new IScalarEvaluator() {
+ return new IRunningAggregateEvaluator() {
+ @Override
+ public void init() {
+ }
@Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ public void step(IFrameTupleReference tuple, IPointable result) {
// Increments the Unique ID value.
for (int i = BINARY_LENGTH - 1; i >= PAYLOAD_START; i--) {
if (++uidBytes[i] != 0) {
@@ -89,5 +86,4 @@
public FunctionIdentifier getIdentifier() {
return BuiltinFunctions.CREATE_QUERY_UID;
}
-
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java
index c218dec..ee65962 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.transaction.management.service.transaction;
+import static org.apache.asterix.transaction.management.service.transaction.TransactionManagementConstants.LogManagerConstants.TERMINAL_LSN;
+
import java.io.IOException;
import java.io.OutputStream;
import java.util.Map;
@@ -101,10 +103,12 @@
final ITransactionContext txnCtx = getTransactionContext(txnId);
try {
if (txnCtx.isWriteTxn()) {
- LogRecord logRecord = new LogRecord();
- TransactionUtil.formJobTerminateLogRecord(txnCtx, logRecord, false);
- txnSubsystem.getLogManager().log(logRecord);
- txnSubsystem.getCheckpointManager().secure(txnId);
+ if (txnCtx.getFirstLSN() != TERMINAL_LSN) {
+ LogRecord logRecord = new LogRecord();
+ TransactionUtil.formJobTerminateLogRecord(txnCtx, logRecord, false);
+ txnSubsystem.getLogManager().log(logRecord);
+ txnSubsystem.getCheckpointManager().secure(txnId);
+ }
txnSubsystem.getRecoveryManager().rollbackTransaction(txnCtx);
txnCtx.setTxnState(ITransactionManager.ABORTED);
}
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index f89f0ac..6067ef9 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1390,7 +1390,7 @@
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
- <version>8.3.0</version>
+ <version>8.5.4</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index bcdda15..7f3b7ec 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -250,11 +250,33 @@
<artifactId>fastutil</artifactId>
<properties>
<!-- fastutil is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
- <!-- see https://github.com/vigna/fastutil/blob/8.1.1/LICENSE-2.0 -->
- <!-- see https://github.com/vigna/fastutil/blob/8.2.2/LICENSE-2.0 -->
- <!-- see https://github.com/vigna/fastutil/blob/8.3.0/LICENSE-2.0 -->
- <license.ignoreMissingEmbeddedLicense>8.1.1,8.2.2,8.3.0</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>8.1.1,8.2.2,8.3.0</license.ignoreMissingEmbeddedNotice>
+ <!-- see https://github.com/vigna/fastutil/blob/8.5.4/LICENSE-2.0 -->
+ <license.ignoreMissingEmbeddedLicense>8.5.4</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>8.5.4</license.ignoreMissingEmbeddedNotice>
+ </properties>
+ </project>
+ </supplement>
+ <supplement>
+ <project>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil-core</artifactId>
+ <properties>
+ <!-- fastutil is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
+ <!-- see https://github.com/vigna/fastutil/blob/8.5.4/LICENSE-2.0 -->
+ <license.ignoreMissingEmbeddedLicense>8.5.4</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>8.5.4</license.ignoreMissingEmbeddedNotice>
+ </properties>
+ </project>
+ </supplement>
+ <supplement>
+ <project>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil-extra</artifactId>
+ <properties>
+ <!-- fastutil is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
+ <!-- see https://github.com/vigna/fastutil/blob/8.5.4/LICENSE-2.0 -->
+ <license.ignoreMissingEmbeddedLicense>8.5.4</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>8.5.4</license.ignoreMissingEmbeddedNotice>
</properties>
</project>
</supplement>
diff --git a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
index 4b8179c..0b7af91 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
+++ b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
@@ -113,6 +113,7 @@
return errorCode;
}
+ @Override
public Serializable[] getParams() {
return params;
}
@@ -121,6 +122,7 @@
return nodeId;
}
+ @Override
public SourceLocation getSourceLocation() {
return sourceLoc;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index 49be31c..c6a79a5 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -106,8 +106,7 @@
expressionRuntimeProvider, expressionTypeComputer, oc, expressionEvalSizeComputer,
partialAggregationTypeComputer, predEvaluatorFactoryProvider,
physicalOptimizationConfig.getFrameSize(), clusterLocations, warningCollector,
- maxWarnings);
-
+ maxWarnings, physicalOptimizationConfig);
PlanCompiler pc = new PlanCompiler(context);
return pc.compilePlan(plan, jobEventListenerFactory);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/pom.xml b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
index b6c19f0..f58a7f6 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
@@ -104,7 +104,7 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
+ <artifactId>fastutil-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
index 313fc30..56040dc 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
@@ -36,6 +36,11 @@
super(variables, expressions);
}
+ public RunningAggregateOperator(LogicalVariable var, Mutable<ILogicalExpression> expr) {
+ this.variables.add(var);
+ this.expressions.add(expr);
+ }
+
@Override
public LogicalOperatorTag getOperatorTag() {
return LogicalOperatorTag.RUNNINGAGGREGATE;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
index aa2ecdc..0df9851 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -487,11 +487,16 @@
public static List<Mutable<ILogicalExpression>> createVariableReferences(Collection<LogicalVariable> varList,
SourceLocation sourceLoc) {
List<Mutable<ILogicalExpression>> varRefs = new ArrayList<>(varList.size());
+ createVariableReferences(varList, sourceLoc, varRefs);
+ return varRefs;
+ }
+
+ public static void createVariableReferences(Collection<LogicalVariable> varList, SourceLocation sourceLoc,
+ List<Mutable<ILogicalExpression>> outVarRefList) {
for (LogicalVariable var : varList) {
VariableReferenceExpression varRef = new VariableReferenceExpression(var);
varRef.setSourceLocation(sourceLoc);
- varRefs.add(new MutableObject<>(varRef));
+ outVarRefList.add(new MutableObject<>(varRef));
}
- return varRefs;
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 944ddf7..20632f1 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.algebricks.core.config;
+import org.apache.hyracks.util.StorageUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -34,4 +35,6 @@
public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true;
public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true;
public static final boolean ARRAY_INDEX_DEFAULT = false;
+ public static final int EXTERNAL_SCAN_BUFFER_SIZE =
+ StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
index 41cde2e..2ca4a36 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
@@ -34,6 +34,7 @@
import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
import org.apache.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
@@ -70,6 +71,7 @@
private final IPartialAggregationTypeComputer partialAggregationTypeComputer;
private final IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider;
private final int frameSize;
+ private final PhysicalOptimizationConfig physicalOptimizationConfig;
private AlgebricksAbsolutePartitionConstraint clusterLocations;
private int varCounter;
private final ITypingContext typingContext;
@@ -90,7 +92,7 @@
IPartialAggregationTypeComputer partialAggregationTypeComputer,
IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider, int frameSize,
AlgebricksAbsolutePartitionConstraint clusterLocations, IWarningCollector warningCollector,
- long maxWarnings) {
+ long maxWarnings, PhysicalOptimizationConfig physicalOptimizationConfig) {
this.outerFlowSchema = outerFlowSchema;
this.metadataProvider = metadataProvider;
this.appContext = appContext;
@@ -113,6 +115,7 @@
this.partialAggregationTypeComputer = partialAggregationTypeComputer;
this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider;
this.frameSize = frameSize;
+ this.physicalOptimizationConfig = physicalOptimizationConfig;
this.varCounter = 0;
this.warningCollector = warningCollector;
this.maxWarnings = maxWarnings;
@@ -227,4 +230,8 @@
public long getMaxWarnings() {
return maxWarnings;
}
+
+ public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
+ return physicalOptimizationConfig;
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index f18cad2..62bf671 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -46,6 +46,7 @@
private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN";
private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
private static final String ARRAY_INDEX = "ARRAY_INDEX";
+ private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
private final Properties properties = new Properties();
@@ -235,6 +236,14 @@
setBoolean(ARRAY_INDEX, arrayIndex);
}
+ public int getExternalScanBufferSize() {
+ return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
+ }
+
+ public void setExternalScanBufferSize(int bufferSize) {
+ setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
index a563f46..956f0b7 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -43,6 +43,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.WindowOperator;
@@ -50,7 +51,7 @@
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
/**
- * Removes unused variables from Assign, Unnest, Aggregate, UnionAll, and Group-by operators.
+ * Removes unused variables from Assign, Unnest, Aggregate, RunningAggregate, UnionAll, and Group-by operators.
*/
public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule {
@@ -223,6 +224,13 @@
isTransformed = true;
}
return agg.getVariables().size();
+ case RUNNINGAGGREGATE:
+ RunningAggregateOperator ragg = (RunningAggregateOperator) op;
+ if (removeUnusedVarsAndExprs(toRemove, ragg.getVariables(), ragg.getExpressions())) {
+ context.computeAndSetTypeEnvironmentForOperator(ragg);
+ isTransformed = true;
+ }
+ return ragg.getVariables().size();
case UNNEST:
UnnestOperator uOp = (UnnestOperator) op;
LogicalVariable pVar = uOp.getPositionalVariable();
@@ -354,6 +362,11 @@
assignVarsSetInThisOp.addAll(agg.getVariables());
targetOpFound = true;
break;
+ case RUNNINGAGGREGATE:
+ RunningAggregateOperator ragg = (RunningAggregateOperator) op;
+ assignVarsSetInThisOp.addAll(ragg.getVariables());
+ targetOpFound = true;
+ break;
case UNNEST:
UnnestOperator uOp = (UnnestOperator) op;
LogicalVariable pVar = uOp.getPositionalVariable();
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
index 0769e18..977e5d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
@@ -130,6 +130,7 @@
return errorCode;
}
+ @Override
public Serializable[] getParams() {
return params;
}
@@ -138,6 +139,7 @@
return nodeId;
}
+ @Override
public SourceLocation getSourceLocation() {
return sourceLoc;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
index 33b3995..b9d2deb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.api.exceptions;
+import java.io.Serializable;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Stream;
@@ -58,6 +59,16 @@
Optional<IError> getError();
/**
+ * @return the source location
+ */
+ SourceLocation getSourceLocation();
+
+ /**
+ * @return the parameters to use when formatting
+ */
+ Serializable[] getParams();
+
+ /**
* Indicates whether this exception matches the supplied error code
*/
default boolean matches(IError candidate) {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
index ec8f8e9..5f01559 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
@@ -24,26 +24,29 @@
import java.io.Serializable;
import java.util.Objects;
+import org.apache.commons.lang3.SerializationUtils;
import org.apache.hyracks.api.util.ErrorMessageUtil;
public class Warning implements Serializable {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private final String component;
private final SourceLocation srcLocation;
private final int code;
private final String message;
+ private final Serializable[] params;
- private Warning(String component, SourceLocation srcLocation, int code, String message) {
+ private Warning(String component, SourceLocation srcLocation, int code, String message, Serializable... params) {
this.component = component;
this.srcLocation = srcLocation;
this.code = code;
this.message = message;
+ this.params = params;
}
public static Warning of(SourceLocation srcLocation, IError code, Serializable... params) {
return new Warning(code.component(), srcLocation, code.intValue(), ErrorMessageUtil
- .formatMessage(code.component(), code.intValue(), code.errorMessage(), srcLocation, params));
+ .formatMessage(code.component(), code.intValue(), code.errorMessage(), srcLocation, params), params);
}
public String getComponent() {
@@ -84,13 +87,25 @@
output.writeInt(code);
output.writeUTF(message);
SourceLocation.writeFields(srcLocation, output);
+ writeParams(output, params);
+ }
+
+ private static void writeParams(DataOutput output, Serializable[] params) throws IOException {
+ byte[] serialize = SerializationUtils.serialize(params);
+ output.writeInt(serialize.length);
+ output.write(serialize);
}
public static Warning create(DataInput input) throws IOException {
String comp = input.readUTF();
int code = input.readInt();
String msg = input.readUTF();
- return new Warning(comp, SourceLocation.create(input), code, msg);
+ SourceLocation sourceLocation = SourceLocation.create(input);
+ int paramsLen = input.readInt();
+ byte[] paramsBytes = new byte[paramsLen];
+ input.readFully(paramsBytes, 0, paramsBytes.length);
+ Serializable[] params = SerializationUtils.deserialize(paramsBytes);
+ return new Warning(comp, sourceLocation, code, msg, params);
}
@Override
@@ -98,4 +113,8 @@
return "Warning{" + "component='" + component + '\'' + ", srcLocation=" + srcLocation + ", code=" + code
+ ", message='" + message + '\'' + '}';
}
+
+ public Serializable[] getParams() {
+ return params;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
index 6ecb677..0b1c5a6 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
@@ -278,7 +278,7 @@
failure = th;
try {
long delayMs = delay.calculate(attempt);
- if (!policy.retry(th) || span.remaining(TimeUnit.MILLISECONDS) < delayMs) {
+ if (!policy.retry(th) || span.elapsed() || span.remaining(TimeUnit.MILLISECONDS) < delayMs) {
onFailure.attemptFailed(action, attempt, true, span, failure);
throw HyracksDataException.create(failure);
} else {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 22b240a..01cb9bf 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -98,6 +98,7 @@
PYTHON_ADDITIONAL_PACKAGES(STRING_ARRAY, new String[0]),
PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true),
PYTHON_ARGS(STRING_ARRAY, (String[]) null),
+ PYTHON_ENV(STRING_ARRAY, (String[]) null),
CREDENTIAL_FILE(
OptionTypes.STRING,
(Function<IApplicationConfig, String>) appConfig -> FileUtil
@@ -245,6 +246,8 @@
return "Python args to pass to Python interpreter";
case PYTHON_CMD_AUTOLOCATE:
return "Whether or not to attempt to automatically set PYTHON_CMD to a usable interpreter";
+ case PYTHON_ENV:
+ return "List of environment variables to set when invoking the Python interpreter for Python UDFs. E.g. FOO=1";
case CREDENTIAL_FILE:
return "Path to HTTP basic credentials";
default:
@@ -621,4 +624,5 @@
public String getCredentialFilePath() {
return getAppConfig().getString(Option.CREDENTIAL_FILE);
}
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
index a11d8de..fb143d9 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
@@ -54,7 +54,7 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
+ <artifactId>fastutil-core</artifactId>
</dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
index dabdd4f..c370b58 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
@@ -75,9 +75,15 @@
}
public void erase() throws HyracksDataException {
- close();
- file.delete();
+ try {
+ close();
+ } finally {
+ eraseClosed();
+ }
+ }
+ public void eraseClosed() {
+ file.delete();
// Make sure we never access the file if it is deleted.
file = null;
handle = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
index a5ad500..03ff72f 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
@@ -20,6 +20,7 @@
import java.io.DataOutput;
import java.nio.ByteBuffer;
+import java.util.BitSet;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameWriter;
@@ -28,6 +29,7 @@
import org.apache.hyracks.api.dataflow.value.IMissingWriter;
import org.apache.hyracks.api.dataflow.value.IPredicateEvaluator;
import org.apache.hyracks.api.dataflow.value.ITuplePairComparator;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -43,6 +45,14 @@
import org.apache.hyracks.dataflow.std.buffermanager.VariableFramePool;
public class NestedLoopJoin {
+ // Note: Min memory budget should be less than {@code AbstractJoinPOperator.MIN_FRAME_LIMIT_FOR_JOIN}
+ // Inner join: 1 frame for the outer input side, 1 frame for the inner input side, 1 frame for the output
+ private static final int MIN_FRAME_BUDGET_INNER_JOIN = 3;
+ // Outer join extra: Add 1 frame for the {@code outerMatchLOJ} bitset
+ private static final int MIN_FRAME_BUDGET_OUTER_JOIN = MIN_FRAME_BUDGET_INNER_JOIN + 1;
+ // Outer join needs 1 bit per each tuple in the outer side buffer
+ private static final int ESTIMATE_AVG_TUPLE_SIZE = 128;
+
private final FrameTupleAccessor accessorInner;
private final FrameTupleAccessor accessorOuter;
private final FrameTupleAppender appender;
@@ -54,30 +64,45 @@
private final boolean isLeftOuter;
private final ArrayTupleBuilder missingTupleBuilder;
private final IPredicateEvaluator predEvaluator;
- private boolean isReversed; //Added for handling correct calling for predicate-evaluator upon recursive calls (in OptimizedHybridHashJoin) that cause role-reversal
+ // Added for handling correct calling for predicate-evaluator upon recursive calls
+ // (in OptimizedHybridHashJoin) that cause role-reversal
+ private final boolean isReversed;
private final BufferInfo tempInfo = new BufferInfo(null, -1, -1);
+ private final BitSet outerMatchLOJ;
public NestedLoopJoin(IHyracksJobletContext jobletContext, FrameTupleAccessor accessorOuter,
- FrameTupleAccessor accessorInner, int memSize, IPredicateEvaluator predEval, boolean isLeftOuter,
+ FrameTupleAccessor accessorInner, int memBudgetInFrames, IPredicateEvaluator predEval, boolean isLeftOuter,
IMissingWriter[] missingWriters) throws HyracksDataException {
+ this(jobletContext, accessorOuter, accessorInner, memBudgetInFrames, predEval, isLeftOuter, missingWriters,
+ false);
+ }
+
+ public NestedLoopJoin(IHyracksJobletContext jobletContext, FrameTupleAccessor accessorOuter,
+ FrameTupleAccessor accessorInner, int memBudgetInFrames, IPredicateEvaluator predEval, boolean isLeftOuter,
+ IMissingWriter[] missingWriters, boolean isReversed) throws HyracksDataException {
this.accessorInner = accessorInner;
this.accessorOuter = accessorOuter;
this.appender = new FrameTupleAppender();
this.outBuffer = new VSizeFrame(jobletContext);
this.innerBuffer = new VSizeFrame(jobletContext);
this.appender.reset(outBuffer, true);
- if (memSize < 3) {
- throw new HyracksDataException("Not enough memory is available for Nested Loop Join");
+
+ int minMemBudgetInFrames = isLeftOuter ? MIN_FRAME_BUDGET_OUTER_JOIN : MIN_FRAME_BUDGET_INNER_JOIN;
+ if (memBudgetInFrames < minMemBudgetInFrames) {
+ throw new HyracksDataException(ErrorCode.INSUFFICIENT_MEMORY);
}
+ int outerBufferMngrMemBudgetInFrames = memBudgetInFrames - minMemBudgetInFrames + 1;
+ int outerBufferMngrMemBudgetInBytes = jobletContext.getInitialFrameSize() * outerBufferMngrMemBudgetInFrames;
this.outerBufferMngr = new VariableFrameMemoryManager(
- new VariableFramePool(jobletContext, jobletContext.getInitialFrameSize() * (memSize - 2)),
- FrameFreeSlotPolicyFactory.createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT, memSize - 2));
+ new VariableFramePool(jobletContext, outerBufferMngrMemBudgetInBytes), FrameFreeSlotPolicyFactory
+ .createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT, outerBufferMngrMemBudgetInFrames));
this.predEvaluator = predEval;
- this.isReversed = false;
-
this.isLeftOuter = isLeftOuter;
if (isLeftOuter) {
+ if (isReversed) {
+ throw new HyracksDataException(ErrorCode.ILLEGAL_STATE, "Outer join cannot reverse roles");
+ }
int innerFieldCount = this.accessorInner.getFieldCount();
missingTupleBuilder = new ArrayTupleBuilder(innerFieldCount);
DataOutput out = missingTupleBuilder.getDataOutput();
@@ -85,9 +110,14 @@
missingWriters[i].writeMissing(out);
missingTupleBuilder.addFieldEndOffset();
}
+ // Outer join needs 1 bit per each tuple in the outer side buffer
+ int outerMatchLOJCardinalityEstimate = outerBufferMngrMemBudgetInBytes / ESTIMATE_AVG_TUPLE_SIZE;
+ outerMatchLOJ = new BitSet(Math.max(outerMatchLOJCardinalityEstimate, 1));
} else {
missingTupleBuilder = null;
+ outerMatchLOJ = null;
}
+ this.isReversed = isReversed;
FileReference file =
jobletContext.createManagedWorkspaceFile(this.getClass().getSimpleName() + this.toString());
@@ -117,23 +147,7 @@
return;
}
if (outerBufferMngr.insertFrame(outerBuffer) < 0) {
- RunFileReader runFileReader = runFileWriter.createReader();
- try {
- runFileReader.open();
- if (runFileReader.nextFrame(innerBuffer)) {
- do {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- blockJoin(outerBufferMngr.getFrame(i, tempInfo), innerBuffer.getBuffer(), writer);
- }
- } while (runFileReader.nextFrame(innerBuffer));
- } else if (isLeftOuter) {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- appendMissing(outerBufferMngr.getFrame(i, tempInfo), writer);
- }
- }
- } finally {
- runFileReader.close();
- }
+ multiBlockJoin(writer);
outerBufferMngr.reset();
if (outerBufferMngr.insertFrame(outerBuffer) < 0) {
throw new HyracksDataException("The given outer frame of size:" + outerBuffer.capacity()
@@ -142,16 +156,51 @@
}
}
- private void blockJoin(BufferInfo outerBufferInfo, ByteBuffer innerBuffer, IFrameWriter writer)
- throws HyracksDataException {
- accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(), outerBufferInfo.getLength());
- accessorInner.reset(innerBuffer);
- int tupleCount0 = accessorOuter.getTupleCount();
- int tupleCount1 = accessorInner.getTupleCount();
+ private void multiBlockJoin(IFrameWriter writer) throws HyracksDataException {
+ int outerBufferFrameCount = outerBufferMngr.getNumFrames();
+ if (outerBufferFrameCount == 0) {
+ return;
+ }
+ RunFileReader runFileReader = runFileWriter.createReader();
+ try {
+ runFileReader.open();
+ if (isLeftOuter) {
+ outerMatchLOJ.clear();
+ }
+ while (runFileReader.nextFrame(innerBuffer)) {
+ int outerTupleRunningCount = 0;
+ for (int i = 0; i < outerBufferFrameCount; i++) {
+ BufferInfo outerBufferInfo = outerBufferMngr.getFrame(i, tempInfo);
+ accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(),
+ outerBufferInfo.getLength());
+ int outerTupleCount = accessorOuter.getTupleCount();
+ accessorInner.reset(innerBuffer.getBuffer());
+ blockJoin(outerTupleRunningCount, writer);
+ outerTupleRunningCount += outerTupleCount;
+ }
+ }
+ if (isLeftOuter) {
+ int outerTupleRunningCount = 0;
+ for (int i = 0; i < outerBufferFrameCount; i++) {
+ BufferInfo outerBufferInfo = outerBufferMngr.getFrame(i, tempInfo);
+ accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(),
+ outerBufferInfo.getLength());
+ int outerFrameTupleCount = accessorOuter.getTupleCount();
+ appendMissing(outerTupleRunningCount, outerFrameTupleCount, writer);
+ outerTupleRunningCount += outerFrameTupleCount;
+ }
+ }
+ } finally {
+ runFileReader.close();
+ }
+ }
- for (int i = 0; i < tupleCount0; ++i) {
+ private void blockJoin(int outerTupleStartPos, IFrameWriter writer) throws HyracksDataException {
+ int outerTupleCount = accessorOuter.getTupleCount();
+ int innerTupleCount = accessorInner.getTupleCount();
+ for (int i = 0; i < outerTupleCount; ++i) {
boolean matchFound = false;
- for (int j = 0; j < tupleCount1; ++j) {
+ for (int j = 0; j < innerTupleCount; ++j) {
int c = tpComparator.compare(accessorOuter, i, accessorInner, j);
boolean prdEval = evaluatePredicate(i, j);
if (c == 0 && prdEval) {
@@ -159,13 +208,8 @@
appendToResults(i, j, writer);
}
}
-
- if (!matchFound && isLeftOuter) {
- final int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
- final byte[] ntByteArray = missingTupleBuilder.getByteArray();
- final int ntSize = missingTupleBuilder.getSize();
- FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0,
- ntSize);
+ if (isLeftOuter && matchFound) {
+ outerMatchLOJ.set(outerTupleStartPos + i);
}
}
}
@@ -191,15 +235,18 @@
FrameUtils.appendConcatToWriter(writer, appender, accessor1, tupleId1, accessor2, tupleId2);
}
- private void appendMissing(BufferInfo outerBufferInfo, IFrameWriter writer) throws HyracksDataException {
- accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(), outerBufferInfo.getLength());
- int tupleCount = accessorOuter.getTupleCount();
- for (int i = 0; i < tupleCount; ++i) {
- final int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
- final byte[] ntByteArray = missingTupleBuilder.getByteArray();
- final int ntSize = missingTupleBuilder.getSize();
- FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0,
- ntSize);
+ private void appendMissing(int outerFrameMngrStartPos, int outerFrameTupleCount, IFrameWriter writer)
+ throws HyracksDataException {
+ int limit = outerFrameMngrStartPos + outerFrameTupleCount;
+ for (int outerTuplePos =
+ outerMatchLOJ.nextClearBit(outerFrameMngrStartPos); outerTuplePos < limit; outerTuplePos =
+ outerMatchLOJ.nextClearBit(outerTuplePos + 1)) {
+ int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
+ byte[] ntByteArray = missingTupleBuilder.getByteArray();
+ int ntSize = missingTupleBuilder.getSize();
+ int outerAccessorTupleIndex = outerTuplePos - outerFrameMngrStartPos;
+ FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, outerAccessorTupleIndex, ntFieldEndOffsets,
+ ntByteArray, 0, ntSize);
}
}
@@ -210,22 +257,10 @@
}
public void completeJoin(IFrameWriter writer) throws HyracksDataException {
- RunFileReader runFileReader = runFileWriter.createDeleteOnCloseReader();
try {
- runFileReader.open();
- if (runFileReader.nextFrame(innerBuffer)) {
- do {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- blockJoin(outerBufferMngr.getFrame(i, tempInfo), innerBuffer.getBuffer(), writer);
- }
- } while (runFileReader.nextFrame(innerBuffer));
- } else if (isLeftOuter) {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- appendMissing(outerBufferMngr.getFrame(i, tempInfo), writer);
- }
- }
+ multiBlockJoin(writer);
} finally {
- runFileReader.close();
+ runFileWriter.eraseClosed();
}
appender.write(writer, true);
}
@@ -233,8 +268,4 @@
public void releaseMemory() throws HyracksDataException {
outerBufferMngr.reset();
}
-
- public void setIsReversed(boolean b) {
- this.isReversed = b;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index c142113..bb79981 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -802,11 +802,10 @@
// The nested loop join result is outer + inner. All the other operator is probe + build.
// Hence the reverse relation is different.
boolean isReversed = outerRd == buildRd && innerRd == probeRd;
- assert isLeftOuter ? !isReversed : true : "LeftOut Join can not reverse roles";
ITuplePairComparator nljComptorOuterInner = isReversed ? buildComp : probComp;
NestedLoopJoin nlj = new NestedLoopJoin(jobletCtx, new FrameTupleAccessor(outerRd),
- new FrameTupleAccessor(innerRd), memorySize, predEvaluator, isLeftOuter, nonMatchWriter);
- nlj.setIsReversed(isReversed);
+ new FrameTupleAccessor(innerRd), memorySize, predEvaluator, isLeftOuter, nonMatchWriter,
+ isReversed);
nlj.setComparator(nljComptorOuterInner);
IFrame cacheBuff = new VSizeFrame(jobletCtx);
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
index fa64003..514a7dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
@@ -22,8 +22,6 @@
import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.concurrent.ConcurrentMap;
@@ -177,12 +175,7 @@
public String localPath(IServletRequest request) {
final String uri = request.getHttpRequest().uri();
int queryStart = uri.indexOf('?');
- String localPath = queryStart == -1 ? uri.substring(trim(uri)) : uri.substring(trim(uri), queryStart);
- try {
- return URLDecoder.decode(localPath, StandardCharsets.UTF_8.name());
- } catch (UnsupportedEncodingException e) {
- throw new IllegalArgumentException(e);
- }
+ return queryStart == -1 ? uri.substring(trim(uri)) : uri.substring(trim(uri), queryStart);
}
public String servletPath(IServletRequest request) {
diff --git a/hyracks-fullstack/hyracks/hyracks-util/pom.xml b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
index 4993443..a03e884 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
@@ -101,7 +101,7 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
+ <artifactId>fastutil-core</artifactId>
</dependency>
</dependencies>
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 0fb88ed..27f9ab1 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -307,8 +307,8 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
- <version>8.3.0</version>
+ <artifactId>fastutil-core</artifactId>
+ <version>8.5.4</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>