fixed issue 731, 740, and more.
commit 0b46141bea8d503896dc06308f102131df2e4f3d
Author: Young-Seok <kisskys@gmail.com>
Date: Tue May 20 12:52:54 2014 -0700
fixed issues of access method rules that try to use incompatible indexes and similarity functions
commit a0ea4e411503de265f1883aa3837a45be4a8747a
Merge: bb8fe91 b5785a9
Author: Young-Seok <kisskys@gmail.com>
Date: Sun May 18 13:00:33 2014 -0700
merged from master branch to kisskys/left-outer-join-issue branch
commit bb8fe91ffd4fec3d495d32442020447693be8548
Author: Young-Seok <kisskys@gmail.com>
Date: Sun May 18 11:33:54 2014 -0700
another fix for picking available index for leftouterjoin plan
commit 60b057ecec6a157e3e11cb316ef7d38601483741
Merge: a743e44 6cb7fd9
Author: Young-Seok <kisskys@gmail.com>
Date: Sun May 11 22:22:42 2014 -0700
merged master to kisskys/left-outer-join-issue branch
commit a743e4493f0f84f7a71e671478592d487e7510e3
Author: Young-Seok <kisskys@gmail.com>
Date: Sun May 11 20:51:50 2014 -0700
changes for left-outer-join to pick available indexes
Change-Id: I0d89d20c6cc076f40d1fbc5687f0b70e49a91eed
Reviewed-on: http://fulliautomatix.ics.uci.edu:8443/33
Reviewed-by: Inci Cetindil <icetindil@gmail.com>
Tested-by: Ian Maxon <imaxon@uci.edu>
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/base/LogicalOperatorDeepCopyVisitor.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/base/LogicalOperatorDeepCopyVisitor.java
index 9fc2a12..9a532b4 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/base/LogicalOperatorDeepCopyVisitor.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/base/LogicalOperatorDeepCopyVisitor.java
@@ -359,7 +359,8 @@
@Override
public ILogicalOperator visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException {
- SelectOperator opCopy = new SelectOperator(exprDeepCopyVisitor.deepCopyExpressionReference(op.getCondition()));
+ SelectOperator opCopy = new SelectOperator(exprDeepCopyVisitor.deepCopyExpressionReference(op.getCondition()),
+ op.getRetainNull(), deepCopyVariable(op.getNullPlaceholderVariable()));
deepCopyInputs(op, opCopy, arg);
copyAnnotations(op, opCopy);
opCopy.setExecutionMode(op.getExecutionMode());
@@ -384,10 +385,11 @@
}
@Override
- public ILogicalOperator visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException {
+ public ILogicalOperator visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
UnnestMapOperator opCopy = new UnnestMapOperator(deepCopyVariableList(op.getVariables()),
- exprDeepCopyVisitor.deepCopyExpressionReference(op.getExpressionRef()),
- op.getVariableTypes(), op.propagatesInput());
+ exprDeepCopyVisitor.deepCopyExpressionReference(op.getExpressionRef()), op.getVariableTypes(),
+ op.propagatesInput());
deepCopyInputs(op, opCopy, arg);
copyAnnotations(op, opCopy);
opCopy.setExecutionMode(op.getExecutionMode());
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/BTreeSearchPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index 6f250e7..d70d9a8 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -113,9 +113,9 @@
VariableUtilities.getLiveVariables(unnestMap, outputVars);
}
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch = metadataProvider.buildBtreeRuntime(
- builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(), dataset,
- jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes, jobGenParams.isLowKeyInclusive(),
- jobGenParams.isHighKeyInclusive(), implConfig);
+ builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(),
+ jobGenParams.getRetainNull(), dataset, jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes,
+ jobGenParams.isLowKeyInclusive(), jobGenParams.isHighKeyInclusive(), implConfig);
builder.contributeHyracksOperator(unnestMap, btreeSearch.first);
builder.contributeAlgebricksPartitionConstraint(btreeSearch.first, btreeSearch.second);
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 28e91ca..db070e7 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -128,8 +128,9 @@
// Build runtime.
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> invIndexSearch = buildInvertedIndexRuntime(
metadataProvider, context, builder.getJobSpec(), unnestMapOp, opSchema, jobGenParams.getRetainInput(),
- jobGenParams.getDatasetName(), dataset, jobGenParams.getIndexName(), jobGenParams.getSearchKeyType(),
- keyIndexes, jobGenParams.getSearchModifierType(), jobGenParams.getSimilarityThreshold());
+ jobGenParams.getRetainNull(), jobGenParams.getDatasetName(), dataset, jobGenParams.getIndexName(),
+ jobGenParams.getSearchKeyType(), keyIndexes, jobGenParams.getSearchModifierType(),
+ jobGenParams.getSimilarityThreshold());
// Contribute operator in hyracks job.
builder.contributeHyracksOperator(unnestMapOp, invIndexSearch.first);
builder.contributeAlgebricksPartitionConstraint(invIndexSearch.first, invIndexSearch.second);
@@ -139,8 +140,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildInvertedIndexRuntime(
AqlMetadataProvider metadataProvider, JobGenContext context, JobSpecification jobSpec,
- UnnestMapOperator unnestMap, IOperatorSchema opSchema, boolean retainInput, String datasetName,
- Dataset dataset, String indexName, ATypeTag searchKeyType, int[] keyFields,
+ UnnestMapOperator unnestMap, IOperatorSchema opSchema, boolean retainInput, boolean retainNull,
+ String datasetName, Dataset dataset, String indexName, ATypeTag searchKeyType, int[] keyFields,
SearchModifierType searchModifierType, IAlgebricksConstantValue similarityThreshold)
throws AlgebricksException {
try {
@@ -239,7 +240,8 @@
jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first,
appContext.getIndexLifecycleManagerProvider(), tokenTypeTraits, tokenComparatorFactories,
invListsTypeTraits, invListsComparatorFactories, dataflowHelperFactory, queryTokenizerFactory,
- searchModifierFactory, outputRecDesc, retainInput, NoOpOperationCallbackFactory.INSTANCE);
+ searchModifierFactory, outputRecDesc, retainInput, retainNull, context.getNullWriterFactory(),
+ NoOpOperationCallbackFactory.INSTANCE);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(invIndexSearchOp,
secondarySplitsAndConstraint.second);
} catch (MetadataException e) {
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/RTreeSearchPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/RTreeSearchPOperator.java
index 705ef1d..33ad11b 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/RTreeSearchPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/RTreeSearchPOperator.java
@@ -82,8 +82,8 @@
VariableUtilities.getLiveVariables(unnestMap, outputVars);
}
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch = mp.buildRtreeRuntime(
- builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(), dataset,
- jobGenParams.getIndexName(), keyIndexes);
+ builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(),
+ jobGenParams.getRetainNull(), dataset, jobGenParams.getIndexName(), keyIndexes);
builder.contributeHyracksOperator(unnestMap, rtreeSearch.first);
builder.contributeAlgebricksPartitionConstraint(rtreeSearch.first, rtreeSearch.second);
ILogicalOperator srcExchange = unnestMap.getInputs().get(0).getValue();
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/AsterixInlineVariablesRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/AsterixInlineVariablesRule.java
index 05051de..8a4e9e2 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/AsterixInlineVariablesRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/AsterixInlineVariablesRule.java
@@ -20,12 +20,18 @@
public class AsterixInlineVariablesRule extends InlineVariablesRule {
public AsterixInlineVariablesRule() {
- // Do not inline field accesses because doing so would interfere with our access method rewrites.
+ // Do not inline field accesses and spatial functions because doing so would interfere with our access method rewrites.
// TODO: For now we must also exclude record constructor functions to avoid breaking our type casting rules
// IntroduceStaticTypeCastRule and IntroduceDynamicTypeCastRule.
doNotInlineFuncs.add(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME);
doNotInlineFuncs.add(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX);
doNotInlineFuncs.add(AsterixBuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR);
doNotInlineFuncs.add(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR);
+ doNotInlineFuncs.add(AsterixBuiltinFunctions.CREATE_CIRCLE);
+ doNotInlineFuncs.add(AsterixBuiltinFunctions.CREATE_LINE);
+ doNotInlineFuncs.add(AsterixBuiltinFunctions.CREATE_MBR);
+ doNotInlineFuncs.add(AsterixBuiltinFunctions.CREATE_POINT);
+ doNotInlineFuncs.add(AsterixBuiltinFunctions.CREATE_POLYGON);
+ doNotInlineFuncs.add(AsterixBuiltinFunctions.CREATE_RECTANGLE);
}
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java
index e0cecf8..02b15b2 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java
@@ -25,23 +25,17 @@
import edu.uci.ics.asterix.om.base.IAObject;
import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ATypeTag;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.types.TypeHelper;
import edu.uci.ics.asterix.optimizer.base.FuzzyUtils;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
@@ -93,67 +87,11 @@
if (fi.equals(AsterixBuiltinFunctions.FUZZY_EQ)) {
List<Mutable<ILogicalExpression>> inputExps = funcExp.getArguments();
- // TODO: Current hack to be able to optimize selections.
- // We change the behavior of this rule for the specific cases of const-var, or for edit-distance functions.
- boolean useExprAsIs = false;
-
String simFuncName = FuzzyUtils.getSimFunction(metadataProvider);
ArrayList<Mutable<ILogicalExpression>> similarityArgs = new ArrayList<Mutable<ILogicalExpression>>();
- List<ATypeTag> inputExprTypes = new ArrayList<ATypeTag>();
- for (int i = 0; i < 2; i++) {
+ for (int i = 0; i < inputExps.size(); ++i) {
Mutable<ILogicalExpression> inputExpRef = inputExps.get(i);
- ILogicalExpression inputExp = inputExpRef.getValue();
-
- // get the input type tag
- ATypeTag inputTypeTag = null;
- if (inputExp.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- VariableReferenceExpression inputVarRef = (VariableReferenceExpression) inputExp;
- LogicalVariable inputVar = inputVarRef.getVariableReference();
- IAType t = TypeHelper.getNonOptionalType((IAType) env.getVarType(inputVar));
- inputExprTypes.add(t.getTypeTag());
- } else if (inputExp.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- // Hack to make sure that we will add the func call as is, without wrapping a tokenizer around.
- IAType type = (IAType) context.getExpressionTypeComputer().getType(inputExp, metadataProvider, env);
- inputTypeTag = type.getTypeTag();
- // Only auto-tokenize strings.
- if (inputTypeTag == ATypeTag.STRING) {
- // Strings will be auto-tokenized.
- inputTypeTag = ATypeTag.UNORDEREDLIST;
- } else {
- useExprAsIs = true;
- }
- inputExprTypes.add(inputTypeTag);
- } else if (inputExp.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
- ConstantExpression inputConst = (ConstantExpression) inputExp;
- AsterixConstantValue constVal = (AsterixConstantValue) inputConst.getValue();
- inputTypeTag = constVal.getObject().getType().getTypeTag();
- inputExprTypes.add(inputTypeTag);
- useExprAsIs = true;
- } else {
- throw new NotImplementedException();
- }
-
- if (simFuncName.equals(FuzzyUtils.EDIT_DISTANCE_FUNCTION_NAME)) {
- useExprAsIs = true;
- }
- }
- // TODO: This second loop is only necessary to implement the hack.
- for (int i = 0; i < inputExprTypes.size(); ++i) {
- Mutable<ILogicalExpression> inputExpRef = inputExps.get(i);
- // TODO: Change Jaccard only to accept sets. We should never have to wrap a tokenizer around.
- // get the tokenizer (if any)
- FunctionIdentifier tokenizer = FuzzyUtils.getTokenizer(inputExprTypes.get(i));
- if (useExprAsIs) {
- similarityArgs.add(inputExpRef);
- } else if (tokenizer == null) {
- similarityArgs.add(inputExpRef);
- } else {
- ArrayList<Mutable<ILogicalExpression>> tokenizerArguments = new ArrayList<Mutable<ILogicalExpression>>();
- tokenizerArguments.add(inputExpRef);
- ScalarFunctionCallExpression tokenizerExp = new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(tokenizer), tokenizerArguments);
- similarityArgs.add(new MutableObject<ILogicalExpression>(tokenizerExp));
- }
+ similarityArgs.add(inputExpRef);
}
FunctionIdentifier simFunctionIdentifier = FuzzyUtils.getFunctionIdentifier(simFuncName);
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java
index 8f0837f..e3f21f0 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java
@@ -352,7 +352,7 @@
getItemExprRef.setValue(ConstantExpression.TRUE);
switch (joinOp.getJoinKind()) {
case INNER: {
- extraSelect = new SelectOperator(expRef);
+ extraSelect = new SelectOperator(expRef, false, null);
extraSelect.getInputs().add(new MutableObject<ILogicalOperator>(outputOp));
outputOp = extraSelect;
break;
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index 4c01708..600295c 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -29,6 +29,7 @@
import edu.uci.ics.asterix.om.base.AString;
import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -80,14 +81,15 @@
}
protected void fillSubTreeIndexExprs(OptimizableOperatorSubTree subTree,
- Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) throws AlgebricksException {
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs, IOptimizationContext context)
+ throws AlgebricksException {
Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
// Check applicability of indexes by access method type.
while (amIt.hasNext()) {
Map.Entry<IAccessMethod, AccessMethodAnalysisContext> entry = amIt.next();
AccessMethodAnalysisContext amCtx = entry.getValue();
// For the current access method type, map variables to applicable indexes.
- fillAllIndexExprs(subTree, amCtx);
+ fillAllIndexExprs(subTree, amCtx, context);
}
}
@@ -205,8 +207,8 @@
* on the function identifier, and an analysis of the function's arguments.
* Updates the analyzedAMs accordingly.
*/
- protected boolean analyzeFunctionExpr(AbstractFunctionCallExpression funcExpr, List<AbstractLogicalOperator> assignsAndUnnests,
- Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
+ protected boolean analyzeFunctionExpr(AbstractFunctionCallExpression funcExpr,
+ List<AbstractLogicalOperator> assignsAndUnnests, Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
if (funcIdent == AlgebricksBuiltinFunctions.AND) {
return false;
@@ -252,7 +254,7 @@
List<Index> datasetIndexes = metadataProvider.getDatasetIndexes(dataset.getDataverseName(),
dataset.getDatasetName());
List<Index> indexCandidates = new ArrayList<Index>();
- // Add an index to the candidates if one of the indexed fields is fieldName.
+ // Add an index to the candidates if one of the indexed fields is fieldName
for (Index index : datasetIndexes) {
if (index.getKeyFieldNames().contains(fieldName)) {
indexCandidates.add(index);
@@ -269,8 +271,8 @@
return true;
}
- protected void fillAllIndexExprs(OptimizableOperatorSubTree subTree, AccessMethodAnalysisContext analysisCtx)
- throws AlgebricksException {
+ protected void fillAllIndexExprs(OptimizableOperatorSubTree subTree, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context) throws AlgebricksException {
for (int optFuncExprIndex = 0; optFuncExprIndex < analysisCtx.matchedFuncExprs.size(); optFuncExprIndex++) {
IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(optFuncExprIndex);
// Try to match variables from optFuncExpr to assigns or unnests.
@@ -295,12 +297,12 @@
}
// Set the fieldName in the corresponding matched function expression.
optFuncExpr.setFieldName(funcVarIndex, fieldName);
+ setTypeTag(context, subTree, optFuncExpr, funcVarIndex);
if (subTree.hasDataSourceScan()) {
fillIndexExprs(fieldName, optFuncExprIndex, subTree.dataset, analysisCtx);
}
}
- }
- else {
+ } else {
UnnestOperator unnestOp = (UnnestOperator) op;
LogicalVariable var = unnestOp.getVariable();
int funcVarIndex = optFuncExpr.findLogicalVar(var);
@@ -317,6 +319,7 @@
}
// Set the fieldName in the corresponding matched function expression.
optFuncExpr.setFieldName(funcVarIndex, fieldName);
+ setTypeTag(context, subTree, optFuncExpr, funcVarIndex);
if (subTree.hasDataSourceScan()) {
fillIndexExprs(fieldName, optFuncExprIndex, subTree.dataset, analysisCtx);
}
@@ -337,17 +340,28 @@
// Set the fieldName in the corresponding matched function expression, and remember matching subtree.
optFuncExpr.setFieldName(funcVarIndex, fieldName);
optFuncExpr.setOptimizableSubTree(funcVarIndex, subTree);
+ setTypeTag(context, subTree, optFuncExpr, funcVarIndex);
if (subTree.hasDataSourceScan()) {
fillIndexExprs(fieldName, optFuncExprIndex, subTree.dataset, analysisCtx);
}
}
}
}
+
+ private void setTypeTag(IOptimizationContext context, OptimizableOperatorSubTree subTree,
+ IOptimizableFuncExpr optFuncExpr, int funcVarIndex) throws AlgebricksException {
+ // Set the typeTag if the type is not null
+ IAType type = (IAType) context.getOutputTypeEnvironment(subTree.root).getVarType(
+ optFuncExpr.getLogicalVar(funcVarIndex));
+ optFuncExpr.setTypeTag(funcVarIndex, type.getTypeTag());
+ }
+
/**
* Returns the field name corresponding to the assigned variable at varIndex.
* Returns null if the expr at varIndex does not yield to a field access function after following a set of allowed functions.
*/
- protected String getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree subTree, int opIndex, int assignVarIndex) {
+ protected String getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree subTree,
+ int opIndex, int assignVarIndex) {
// Get expression corresponding to opVar at varIndex.
AbstractLogicalExpression expr = null;
AbstractLogicalOperator op = subTree.assignsAndUnnests.get(opIndex);
@@ -387,8 +401,7 @@
int fieldIndex = ((AInt32) ((AsterixConstantValue) constExpr.getValue()).getObject()).getIntegerValue();
return subTree.recordType.getFieldNames()[fieldIndex];
}
- if (funcIdent != AsterixBuiltinFunctions.WORD_TOKENS
- && funcIdent != AsterixBuiltinFunctions.GRAM_TOKENS
+ if (funcIdent != AsterixBuiltinFunctions.WORD_TOKENS && funcIdent != AsterixBuiltinFunctions.GRAM_TOKENS
&& funcIdent != AsterixBuiltinFunctions.SUBSTRING
&& funcIdent != AsterixBuiltinFunctions.SUBSTRING_BEFORE
&& funcIdent != AsterixBuiltinFunctions.SUBSTRING_AFTER) {
@@ -416,8 +429,7 @@
return getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, varIndex);
}
}
- }
- else {
+ } else {
UnnestOperator unnestOp = (UnnestOperator) curOp;
LogicalVariable var = unnestOp.getVariable();
if (var.equals(curVar)) {
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
index ae0d48c..9d121a7a 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
@@ -18,8 +18,12 @@
import java.util.HashMap;
import java.util.List;
+import org.apache.commons.lang3.mutable.Mutable;
+
import edu.uci.ics.asterix.metadata.entities.Dataset;
import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
/**
* Context for analyzing the applicability of a single access method.
@@ -36,6 +40,10 @@
// Maps from index to the dataset it is indexing.
public HashMap<Index, Dataset> indexDatasetMap = new HashMap<Index, Dataset>();
+ // variables for resetting null placeholder for left-outer-join
+ private Mutable<ILogicalOperator> lojGroupbyOpRef = null;
+ private ScalarFunctionCallExpression lojIsNullFuncInGroupBy = null;
+
public void addIndexExpr(Dataset dataset, Index index, Integer exprIndex) {
List<Integer> exprs = indexExprs.get(index);
if (exprs == null) {
@@ -49,4 +57,21 @@
public List<Integer> getIndexExprs(Index index) {
return indexExprs.get(index);
}
+
+ public void setLOJGroupbyOpRef(Mutable<ILogicalOperator> opRef) {
+ lojGroupbyOpRef = opRef;
+ }
+
+ public Mutable<ILogicalOperator> getLOJGroupbyOpRef() {
+ return lojGroupbyOpRef;
+ }
+
+ public void setLOJIsNullFuncInGroupBy(ScalarFunctionCallExpression isNullFunc) {
+ lojIsNullFuncInGroupBy = isNullFunc;
+ }
+
+ public ScalarFunctionCallExpression getLOJIsNullFuncInGroupBy() {
+ return lojIsNullFuncInGroupBy;
+ }
+
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java
index 4e24c26..9abaefa 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java
@@ -37,21 +37,23 @@
protected String dataverseName;
protected String datasetName;
protected boolean retainInput;
+ protected boolean retainNull;
protected boolean requiresBroadcast;
protected boolean isPrimaryIndex;
- private final int NUM_PARAMS = 6;
+ private final int NUM_PARAMS = 7;
public AccessMethodJobGenParams() {
}
public AccessMethodJobGenParams(String indexName, IndexType indexType, String dataverseName, String datasetName,
- boolean retainInput, boolean requiresBroadcast) {
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast) {
this.indexName = indexName;
this.indexType = indexType;
this.dataverseName = dataverseName;
this.datasetName = datasetName;
this.retainInput = retainInput;
+ this.retainNull = retainNull;
this.requiresBroadcast = requiresBroadcast;
this.isPrimaryIndex = datasetName.equals(indexName);
}
@@ -62,6 +64,7 @@
funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createStringConstant(dataverseName)));
funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createStringConstant(datasetName)));
funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createBooleanConstant(retainInput)));
+ funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createBooleanConstant(retainNull)));
funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createBooleanConstant(requiresBroadcast)));
}
@@ -71,7 +74,8 @@
dataverseName = AccessMethodUtils.getStringConstant(funcArgs.get(2));
datasetName = AccessMethodUtils.getStringConstant(funcArgs.get(3));
retainInput = AccessMethodUtils.getBooleanConstant(funcArgs.get(4));
- requiresBroadcast = AccessMethodUtils.getBooleanConstant(funcArgs.get(5));
+ retainNull = AccessMethodUtils.getBooleanConstant(funcArgs.get(5));
+ requiresBroadcast = AccessMethodUtils.getBooleanConstant(funcArgs.get(6));
isPrimaryIndex = datasetName.equals(indexName);
}
@@ -94,6 +98,10 @@
public boolean getRetainInput() {
return retainInput;
}
+
+ public boolean getRetainNull() {
+ return retainNull;
+ }
public boolean getRequiresBroadcast() {
return requiresBroadcast;
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
index f5e6378..1691f0c 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -42,6 +42,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -49,17 +50,24 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
/**
* Static helper functions for rewriting plans using indexes.
*/
public class AccessMethodUtils {
+
public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, List<Object> target)
throws IOException {
ARecordType recordType = (ARecordType) itemType;
@@ -226,7 +234,8 @@
}
}
- public static List<LogicalVariable> getPrimaryKeyVarsFromSecondaryUnnestMap(Dataset dataset, ILogicalOperator unnestMapOp) {
+ public static List<LogicalVariable> getPrimaryKeyVarsFromSecondaryUnnestMap(Dataset dataset,
+ ILogicalOperator unnestMapOp) {
int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
List<LogicalVariable> primaryKeyVars = new ArrayList<LogicalVariable>();
List<LogicalVariable> sourceVars = ((UnnestMapOperator) unnestMapOp).getVariables();
@@ -239,7 +248,8 @@
return primaryKeyVars;
}
- public static List<LogicalVariable> getPrimaryKeyVarsFromPrimaryUnnestMap(Dataset dataset, ILogicalOperator unnestMapOp) {
+ public static List<LogicalVariable> getPrimaryKeyVarsFromPrimaryUnnestMap(Dataset dataset,
+ ILogicalOperator unnestMapOp) {
int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
List<LogicalVariable> primaryKeyVars = new ArrayList<LogicalVariable>();
List<LogicalVariable> sourceVars = ((UnnestMapOperator) unnestMapOp).getVariables();
@@ -309,8 +319,9 @@
public static UnnestMapOperator createPrimaryIndexUnnestMap(DataSourceScanOperator dataSourceScan, Dataset dataset,
ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context, boolean sortPrimaryKeys,
- boolean retainInput, boolean requiresBroadcast) throws AlgebricksException {
- List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, inputOp);
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast) throws AlgebricksException {
+ List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
+ inputOp);
// Optionally add a sort on the primary-index keys before searching the primary index.
OrderOperator order = null;
if (sortPrimaryKeys) {
@@ -329,12 +340,13 @@
// The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments.
List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
BTreeJobGenParams jobGenParams = new BTreeJobGenParams(dataset.getDatasetName(), IndexType.BTREE,
- dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
+ dataset.getDataverseName(), dataset.getDatasetName(), retainInput, retainNull, requiresBroadcast);
// Set low/high inclusive to true for a point lookup.
jobGenParams.setLowKeyInclusive(true);
jobGenParams.setHighKeyInclusive(true);
jobGenParams.setLowKeyVarList(primaryKeyVars, 0, primaryKeyVars.size());
jobGenParams.setHighKeyVarList(primaryKeyVars, 0, primaryKeyVars.size());
+ jobGenParams.setIsEqCondition(true);
jobGenParams.writeToFuncArgs(primaryIndexFuncArgs);
// Variables and types coming out of the primary-index search.
List<LogicalVariable> primaryIndexUnnestVars = new ArrayList<LogicalVariable>();
@@ -365,4 +377,58 @@
return primaryIndexUnnestOp;
}
+ public static ScalarFunctionCallExpression findLOJIsNullFuncInGroupBy(GroupByOperator lojGroupbyOp)
+ throws AlgebricksException {
+ //find IS_NULL function of which argument has the nullPlaceholder variable in the nested plan of groupby.
+ ALogicalPlanImpl subPlan = (ALogicalPlanImpl) lojGroupbyOp.getNestedPlans().get(0);
+ Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0);
+ AbstractLogicalOperator subPlanRootOp = (AbstractLogicalOperator) subPlanRootOpRef.getValue();
+ boolean foundSelectNonNull = false;
+ ScalarFunctionCallExpression isNullFuncExpr = null;
+ AbstractLogicalOperator inputOp = subPlanRootOp;
+ while (inputOp != null) {
+ if (inputOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ SelectOperator selectOp = (SelectOperator) inputOp;
+ if (selectOp.getCondition().getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ if (((AbstractFunctionCallExpression) selectOp.getCondition().getValue()).getFunctionIdentifier()
+ .equals(AlgebricksBuiltinFunctions.NOT)) {
+ ScalarFunctionCallExpression notFuncExpr = (ScalarFunctionCallExpression) selectOp
+ .getCondition().getValue();
+ if (notFuncExpr.getArguments().get(0).getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ if (((AbstractFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue())
+ .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
+ isNullFuncExpr = (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0)
+ .getValue();
+ if (isNullFuncExpr.getArguments().get(0).getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ foundSelectNonNull = true;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ inputOp = inputOp.getInputs().size() > 0 ? (AbstractLogicalOperator) inputOp.getInputs().get(0).getValue()
+ : null;
+ }
+
+ if (!foundSelectNonNull) {
+ throw new AlgebricksException(
+ "Could not find the non-null select operator in GroupByOperator for LEFTOUTERJOIN plan optimization.");
+ }
+ return isNullFuncExpr;
+ }
+
+ public static void resetLOJNullPlaceholderVariableInGroupByOp(AccessMethodAnalysisContext analysisCtx,
+ LogicalVariable newNullPlaceholderVaraible, IOptimizationContext context) throws AlgebricksException {
+
+ //reset the null placeholder variable in groupby operator
+ ScalarFunctionCallExpression isNullFuncExpr = analysisCtx.getLOJIsNullFuncInGroupBy();
+ isNullFuncExpr.getArguments().clear();
+ isNullFuncExpr.getArguments().add(
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(newNullPlaceholderVaraible)));
+
+ //recompute type environment.
+ OperatorPropertiesUtil.typeOpRec(analysisCtx.getLOJGroupbyOpRef(), context);
+ }
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 5132f55..9fadef7 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -87,8 +87,8 @@
}
@Override
- public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr, List<AbstractLogicalOperator> assignsAndUnnests,
- AccessMethodAnalysisContext analysisCtx) {
+ public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr,
+ List<AbstractLogicalOperator> assignsAndUnnests, AccessMethodAnalysisContext analysisCtx) {
boolean matches = AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx);
if (!matches) {
matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVars(funcExpr, analysisCtx);
@@ -114,11 +114,12 @@
SelectOperator select = (SelectOperator) selectRef.getValue();
Mutable<ILogicalExpression> conditionRef = select.getCondition();
ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(selectRef, conditionRef, subTree, null,
- chosenIndex, analysisCtx, false, false, context);
+ chosenIndex, analysisCtx, false, false, false, context);
if (primaryIndexUnnestOp == null) {
return false;
}
- Mutable<ILogicalOperator> opRef = (subTree.assignsAndUnnestsRefs.isEmpty()) ? null : subTree.assignsAndUnnestsRefs.get(0);
+ Mutable<ILogicalOperator> opRef = (subTree.assignsAndUnnestsRefs.isEmpty()) ? null
+ : subTree.assignsAndUnnestsRefs.get(0);
ILogicalOperator op = null;
if (opRef != null) {
op = opRef.getValue();
@@ -147,7 +148,8 @@
@Override
public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef,
OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException {
+ AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin)
+ throws AlgebricksException {
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
Mutable<ILogicalExpression> conditionRef = joinOp.getCondition();
// Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
@@ -155,7 +157,8 @@
// Determine probe and index subtrees based on chosen index.
OptimizableOperatorSubTree indexSubTree = null;
OptimizableOperatorSubTree probeSubTree = null;
- if (leftSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(leftSubTree.dataset.getDatasetName())) {
+ if (!isLeftOuterJoin && leftSubTree.hasDataSourceScan()
+ && dataset.getDatasetName().equals(leftSubTree.dataset.getDatasetName())) {
indexSubTree = leftSubTree;
probeSubTree = rightSubTree;
} else if (rightSubTree.hasDataSourceScan()
@@ -163,15 +166,33 @@
indexSubTree = rightSubTree;
probeSubTree = leftSubTree;
}
+ if (indexSubTree == null) {
+ //This may happen for left outer join case
+ return false;
+ }
+
+ LogicalVariable newNullPlaceHolderVar = null;
+ if (isLeftOuterJoin) {
+ //get a new null place holder variable that is the first field variable of the primary key
+ //from the indexSubTree's datasourceScanOp
+ newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
+ }
+
ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(joinRef, conditionRef, indexSubTree,
- probeSubTree, chosenIndex, analysisCtx, true, true, context);
+ probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context);
if (primaryIndexUnnestOp == null) {
return false;
}
+
+ if (isLeftOuterJoin) {
+ //reset the null place holder variable
+ AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+ }
+
// If there are conditions left, add a new select operator on top.
indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp);
if (conditionRef.getValue() != null) {
- SelectOperator topSelect = new SelectOperator(conditionRef);
+ SelectOperator topSelect = new SelectOperator(conditionRef, isLeftOuterJoin, newNullPlaceHolderVar);
topSelect.getInputs().add(indexSubTree.rootRef);
topSelect.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(topSelect);
@@ -186,7 +207,8 @@
private ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalOperator> topOpRef,
Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
- boolean retainInput, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
+ throws AlgebricksException {
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
// we made sure indexSubTree has datasource scan
@@ -392,7 +414,7 @@
keyVarList, context);
BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE,
- dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
+ dataset.getDataverseName(), dataset.getDatasetName(), retainInput, retainNull, requiresBroadcast);
jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]);
jobGenParams.setHighKeyInclusive(highKeyInclusive[0]);
jobGenParams.setIsEqCondition(isEqCondition);
@@ -416,11 +438,11 @@
chosenIndex, inputOp, jobGenParams, context, false, retainInput);
// Generate the rest of the upstream plan which feeds the search results into the primary index.
- UnnestMapOperator primaryIndexUnnestOp;
- boolean isPrimaryIndex = chosenIndex.getIndexName().equals(dataset.getDatasetName());
+ UnnestMapOperator primaryIndexUnnestOp = null;
+ boolean isPrimaryIndex = chosenIndex.isPrimaryIndex();
if (!isPrimaryIndex) {
primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset, recordType,
- secondaryIndexUnnestOp, context, true, retainInput, false);
+ secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
// Replace the datasource scan with the new plan rooted at
// primaryIndexUnnestMap.
@@ -448,6 +470,7 @@
}
}
}
+
return primaryIndexUnnestOp;
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeJobGenParams.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeJobGenParams.java
index ab15c9e..f7829fa 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeJobGenParams.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeJobGenParams.java
@@ -43,8 +43,8 @@
}
public BTreeJobGenParams(String indexName, IndexType indexType, String dataverseName, String datasetName,
- boolean retainInput, boolean requiresBroadcast) {
- super(indexName, indexType, dataverseName, datasetName, retainInput, requiresBroadcast);
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast) {
+ super(indexName, indexType, dataverseName, datasetName, retainInput, retainNull, requiresBroadcast);
}
public void setLowKeyVarList(List<LogicalVariable> keyVarList, int startIndex, int numKeys) {
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IAccessMethod.java
index 0be4204..1419f33 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IAccessMethod.java
@@ -25,7 +25,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
/**
* Interface that an access method should implement to work with the rewrite
@@ -52,8 +51,8 @@
* @return true if funcExpr is optimizable by this access method, false
* otherwise
*/
- public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr, List<AbstractLogicalOperator> assignsAndUnnests,
- AccessMethodAnalysisContext analysisCtx);
+ public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr,
+ List<AbstractLogicalOperator> assignsAndUnnests, AccessMethodAnalysisContext analysisCtx);
/**
* Indicates whether all index expressions must be matched in order for this
@@ -83,7 +82,8 @@
*/
public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef,
OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException;
+ AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin)
+ throws AlgebricksException;
/**
* Analyzes expr to see whether it is optimizable by the given concrete index.
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java
index 4d43375..dbe5854 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.asterix.optimizer.rules.am;
+import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
@@ -51,4 +52,8 @@
public void setPartialField(boolean partialField);
public boolean containsPartialField();
+
+ public void setTypeTag(int index, ATypeTag typeTag);
+
+ public ATypeTag getTypeTag(int index);
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
index 32d459d..025c2e2 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
@@ -30,9 +30,13 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
/**
@@ -53,14 +57,24 @@
* 3. Check metadata to see if there are applicable indexes.
* 4. Choose an index to apply (for now only a single index will be chosen).
* 5. Rewrite plan using index (delegated to IAccessMethods).
+ * For left-outer-join, additional patterns are checked and additional treatment is needed as follows:
+ * 1. Since left-outer-join operators always have groupby operator on top of it,
+ * first it checks a pattern: (groupby) <-- (leftouterjoin)
+ * 2. Inherently, only the right-subtree of the lojOp can be used as indexSubtree.
+ * So, the right-subtree must have at least one applicable index on join field(s)
+ * 3. The null placeholder variable introduced in groupByOp should be taken care of correctly.
+ * Here, the primary key variable from datasourceScanOp replaces the introduced null placeholder variable.
+ * If the primary key is composite key, then the first variable of the primary key variables becomes the
+ * null place holder variable. This null placeholder variable works for all three types of indexes.
*/
public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethodRule {
protected Mutable<ILogicalOperator> joinRef = null;
- protected InnerJoinOperator join = null;
+ protected AbstractBinaryJoinOperator join = null;
protected AbstractFunctionCallExpression joinCond = null;
protected final OptimizableOperatorSubTree leftSubTree = new OptimizableOperatorSubTree();
protected final OptimizableOperatorSubTree rightSubTree = new OptimizableOperatorSubTree();
+ protected boolean isLeftOuterJoin = false;
// Register access methods.
protected static Map<FunctionIdentifier, List<IAccessMethod>> accessMethods = new HashMap<FunctionIdentifier, List<IAccessMethod>>();
@@ -108,10 +122,10 @@
return false;
}
if (checkLeftSubTreeMetadata) {
- fillSubTreeIndexExprs(leftSubTree, analyzedAMs);
+ fillSubTreeIndexExprs(leftSubTree, analyzedAMs, context);
}
if (checkRightSubTreeMetadata) {
- fillSubTreeIndexExprs(rightSubTree, analyzedAMs);
+ fillSubTreeIndexExprs(rightSubTree, analyzedAMs, context);
}
pruneIndexCandidates(analyzedAMs);
@@ -124,8 +138,17 @@
// Apply plan transformation using chosen index.
AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first);
+
+ //For LOJ, prepare objects to reset LOJ nullPlaceHolderVariable in GroupByOp
+ if (isLeftOuterJoin) {
+ analysisCtx.setLOJGroupbyOpRef(opRef);
+ ScalarFunctionCallExpression isNullFuncExpr = AccessMethodUtils
+ .findLOJIsNullFuncInGroupBy((GroupByOperator) opRef.getValue());
+ analysisCtx.setLOJIsNullFuncInGroupBy(isNullFuncExpr);
+ }
+
boolean res = chosenIndex.first.applyJoinPlanTransformation(joinRef, leftSubTree, rightSubTree,
- chosenIndex.second, analysisCtx, context);
+ chosenIndex.second, analysisCtx, context, isLeftOuterJoin);
if (res) {
OperatorPropertiesUtil.typeOpRec(opRef, context);
}
@@ -139,20 +162,31 @@
if (context.checkIfInDontApplySet(this, op1)) {
return false;
}
- if (op1.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+
+ boolean isInnerJoin = isInnerJoin(op1);
+ isLeftOuterJoin = isLeftOuterJoin(op1);
+
+ if (!isInnerJoin && !isLeftOuterJoin) {
return false;
}
+
// Set and analyze select.
- joinRef = opRef;
- join = (InnerJoinOperator) op1;
+ if (isInnerJoin) {
+ joinRef = opRef;
+ join = (InnerJoinOperator) op1;
+ } else {
+ joinRef = op1.getInputs().get(0);
+ join = (LeftOuterJoinOperator) joinRef.getValue();
+ }
+
// Check that the select's condition is a function call.
ILogicalExpression condExpr = join.getCondition().getValue();
if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
return false;
}
joinCond = (AbstractFunctionCallExpression) condExpr;
- leftSubTree.initFromSubTree(op1.getInputs().get(0));
- rightSubTree.initFromSubTree(op1.getInputs().get(1));
+ leftSubTree.initFromSubTree(join.getInputs().get(0));
+ rightSubTree.initFromSubTree(join.getInputs().get(1));
// One of the subtrees must have a datasource scan.
if (leftSubTree.hasDataSourceScan() || rightSubTree.hasDataSourceScan()) {
return true;
@@ -160,6 +194,15 @@
return false;
}
+ private boolean isLeftOuterJoin(AbstractLogicalOperator op1) {
+ return (op1.getOperatorTag() == LogicalOperatorTag.GROUP && ((AbstractLogicalOperator) op1.getInputs().get(0)
+ .getValue()).getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN);
+ }
+
+ private boolean isInnerJoin(AbstractLogicalOperator op1) {
+ return op1.getOperatorTag() == LogicalOperatorTag.INNERJOIN;
+ }
+
@Override
public Map<FunctionIdentifier, List<IAccessMethod>> getAccessMethods() {
return accessMethods;
@@ -169,5 +212,6 @@
joinRef = null;
join = null;
joinCond = null;
+ isLeftOuterJoin = false;
}
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
index 28ec41d..0f0db4c 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
@@ -98,7 +98,7 @@
return false;
}
- fillSubTreeIndexExprs(subTree, analyzedAMs);
+ fillSubTreeIndexExprs(subTree, analyzedAMs, context);
pruneIndexCandidates(analyzedAMs);
// Choose index to be applied.
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index d73f437..5042f80 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -28,7 +28,6 @@
import edu.uci.ics.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.formats.nontagged.AqlBinaryTokenizerFactoryProvider;
-import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
import edu.uci.ics.asterix.metadata.entities.Dataset;
import edu.uci.ics.asterix.metadata.entities.Index;
import edu.uci.ics.asterix.om.base.AFloat;
@@ -59,6 +58,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
@@ -120,8 +120,13 @@
@Override
public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr,
List<AbstractLogicalOperator> assignsAndUnnests, AccessMethodAnalysisContext analysisCtx) {
+
if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.CONTAINS) {
- return AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx);
+ boolean matches = AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx);
+ if (!matches) {
+ matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVars(funcExpr, analysisCtx);
+ }
+ return matches;
}
return analyzeGetItemFuncExpr(funcExpr, assignsAndUnnests, analysisCtx);
}
@@ -335,7 +340,8 @@
private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex, IOptimizableFuncExpr optFuncExpr,
- boolean retainInput, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
+ throws AlgebricksException {
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
// we made sure indexSubTree has datasource scan
@@ -343,7 +349,7 @@
InvertedIndexJobGenParams jobGenParams = new InvertedIndexJobGenParams(chosenIndex.getIndexName(),
chosenIndex.getIndexType(), dataset.getDataverseName(), dataset.getDatasetName(), retainInput,
- requiresBroadcast);
+ retainNull, requiresBroadcast);
// Add function-specific args such as search modifier, and possibly a similarity threshold.
addFunctionSpecificArgs(optFuncExpr, jobGenParams);
// Add the type of search key from the optFuncExpr.
@@ -374,9 +380,11 @@
jobGenParams.setKeyVarList(keyVarList);
UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
chosenIndex, inputOp, jobGenParams, context, true, retainInput);
+
// Generate the rest of the upstream plan which feeds the search results into the primary index.
UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset,
- recordType, secondaryIndexUnnestOp, context, true, retainInput, false);
+ recordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
+
return primaryIndexUnnestOp;
}
@@ -401,7 +409,7 @@
IOptimizationContext context) throws AlgebricksException {
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr, false,
- false, context);
+ false, false, context);
// Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
subTree.dataSourceRef.setValue(indexPlanRootOp);
return true;
@@ -410,13 +418,15 @@
@Override
public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef,
OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException {
+ AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin)
+ throws AlgebricksException {
// Figure out if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex);
// Determine probe and index subtrees based on chosen index.
OptimizableOperatorSubTree indexSubTree = null;
OptimizableOperatorSubTree probeSubTree = null;
- if (leftSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(leftSubTree.dataset.getDatasetName())) {
+ if (!isLeftOuterJoin && leftSubTree.hasDataSourceScan()
+ && dataset.getDatasetName().equals(leftSubTree.dataset.getDatasetName())) {
indexSubTree = leftSubTree;
probeSubTree = rightSubTree;
} else if (rightSubTree.hasDataSourceScan()
@@ -424,15 +434,33 @@
indexSubTree = rightSubTree;
probeSubTree = leftSubTree;
}
+ if (indexSubTree == null) {
+ //This may happen for left outer join case
+ return false;
+ }
+
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
- // The arguments of edit-distance-contains() function are asymmetrical, we can only use index if the dataset of index subtree and the dataset of first argument's subtree is the same
+ // The arguments of edit-distance-contains() function are asymmetrical, we can only use index
+ // if the dataset of index subtree and the dataset of first argument's subtree is the same
if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS
&& optFuncExpr.getOperatorSubTree(0).dataset != null
&& !optFuncExpr.getOperatorSubTree(0).dataset.getDatasetName().equals(
indexSubTree.dataset.getDatasetName())) {
return false;
}
- InnerJoinOperator join = (InnerJoinOperator) joinRef.getValue();
+
+ //if LOJ, reset null place holder variable
+ LogicalVariable newNullPlaceHolderVar = null;
+ if (isLeftOuterJoin) {
+ //get a new null place holder variable that is the first field variable of the primary key
+ //from the indexSubTree's datasourceScanOp
+ newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
+
+ //reset the null place holder variable
+ AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+ }
+
+ AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) joinRef.getValue();
// Remember the original probe subtree, and its primary-key variables,
// so we can later retrieve the missing attributes via an equi join.
@@ -465,11 +493,12 @@
}
// Create regular indexed-nested loop join path.
ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex,
- optFuncExpr, true, true, context);
+ optFuncExpr, true, isLeftOuterJoin, true, context);
indexSubTree.dataSourceRef.setValue(indexPlanRootOp);
// Change join into a select with the same condition.
- SelectOperator topSelect = new SelectOperator(new MutableObject<ILogicalExpression>(joinCond));
+ SelectOperator topSelect = new SelectOperator(new MutableObject<ILogicalExpression>(joinCond), isLeftOuterJoin,
+ newNullPlaceHolderVar);
topSelect.getInputs().add(indexSubTree.rootRef);
topSelect.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(topSelect);
@@ -650,7 +679,7 @@
// Replace the inputs of the given join op, and replace variables in its
// condition since we deep-copied one of the scanner subtrees which
// changed variables.
- InnerJoinOperator joinOp = (InnerJoinOperator) joinRef.getValue();
+ AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
for (Map.Entry<LogicalVariable, LogicalVariable> entry : copyVarMap.entrySet()) {
joinOp.getCondition().getValue().substituteVar(entry.getKey(), entry.getValue());
}
@@ -707,7 +736,7 @@
}
}
SelectOperator isFilterableSelectOp = new SelectOperator(
- new MutableObject<ILogicalExpression>(isFilterableExpr));
+ new MutableObject<ILogicalExpression>(isFilterableExpr), false, null);
isFilterableSelectOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
isFilterableSelectOp.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(isFilterableSelectOp);
@@ -718,7 +747,7 @@
ILogicalExpression isNotFilterableExpr = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NOT), isNotFilterableArgs);
SelectOperator isNotFilterableSelectOp = new SelectOperator(new MutableObject<ILogicalExpression>(
- isNotFilterableExpr));
+ isNotFilterableExpr), false, null);
isNotFilterableSelectOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
isNotFilterableSelectOp.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(isNotFilterableSelectOp);
@@ -796,88 +825,227 @@
.containsKey(SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE)) {
return false;
}
+
if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CHECK
|| optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) {
- // Must be for a join query.
- if (optFuncExpr.getNumConstantVals() == 1) {
- return true;
- }
- // Check for panic in selection query.
- // TODO: Panic also depends on prePost which is currently hardcoded to be true.
- AsterixConstantValue listOrStrConstVal = (AsterixConstantValue) optFuncExpr.getConstantVal(0);
- AsterixConstantValue intConstVal = (AsterixConstantValue) optFuncExpr.getConstantVal(1);
- IAObject listOrStrObj = listOrStrConstVal.getObject();
- IAObject intObj = intConstVal.getObject();
- AInt32 edThresh = (AInt32) intObj;
- int mergeThreshold = 0;
- // We can only optimize edit distance on strings using an ngram index.
- if (listOrStrObj.getType().getTypeTag() == ATypeTag.STRING
- && (index.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)) {
- AString astr = (AString) listOrStrObj;
- // Compute merge threshold depending on the query grams contain pre- and postfixing
- if (optFuncExpr.containsPartialField()) {
- mergeThreshold = (astr.getStringValue().length() - index.getGramLength() + 1)
- - edThresh.getIntegerValue() * index.getGramLength();
- } else {
- mergeThreshold = (astr.getStringValue().length() + index.getGramLength() - 1)
- - edThresh.getIntegerValue() * index.getGramLength();
- }
- }
- // We can only optimize edit distance on lists using a word index.
- if ((listOrStrObj.getType().getTypeTag() == ATypeTag.ORDEREDLIST || listOrStrObj.getType().getTypeTag() == ATypeTag.UNORDEREDLIST)
- && (index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX)) {
- IACollection alist = (IACollection) listOrStrObj;
- // Compute merge threshold.
- mergeThreshold = alist.size() - edThresh.getIntegerValue();
- }
- if (mergeThreshold <= 0) {
- // We cannot use index to optimize expr.
- return false;
- }
+ return isEditDistanceFuncOptimizable(index, optFuncExpr);
+ }
+
+ if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.SIMILARITY_JACCARD_CHECK) {
+ return isJaccardFuncOptimizable(index, optFuncExpr);
+ }
+
+ if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.CONTAINS) {
+ return isContainsFuncOptimizable(index, optFuncExpr);
+ }
+
+ return false;
+ }
+
+ private boolean isEditDistanceFuncOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
+ if (optFuncExpr.getNumConstantVals() == 1) {
+ return isEditDistanceFuncJoinOptimizable(index, optFuncExpr);
+ } else {
+ return isEditDistanceFuncSelectOptimizable(index, optFuncExpr);
+ }
+ }
+
+ private boolean isEditDistanceFuncJoinOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
+ return isEditDistanceFuncCompatible(optFuncExpr.getTypeTag(0), index.getIndexType());
+ }
+
+ private boolean isEditDistanceFuncCompatible(ATypeTag typeTag, IndexType indexType) {
+ // We can only optimize edit distance on strings using an ngram index.
+ if (typeTag == ATypeTag.STRING
+ && (indexType == IndexType.SINGLE_PARTITION_NGRAM_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)) {
return true;
}
- // TODO: We need more checking: gram length, prePost, etc.
- if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.SIMILARITY_JACCARD_CHECK) {
- // Check the tokenization function of the non-constant func arg to see if it fits the concrete index type.
- ILogicalExpression arg1 = optFuncExpr.getFuncExpr().getArguments().get(0).getValue();
- ILogicalExpression arg2 = optFuncExpr.getFuncExpr().getArguments().get(1).getValue();
- ILogicalExpression nonConstArg = null;
- if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- nonConstArg = arg1;
+ // We can only optimize edit distance on lists using a word index.
+ if ((typeTag == ATypeTag.ORDEREDLIST)
+ && (indexType == IndexType.SINGLE_PARTITION_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX)) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean isEditDistanceFuncSelectOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
+
+ // Check for panic in selection query.
+ // TODO: Panic also depends on prePost which is currently hardcoded to be true.
+ AsterixConstantValue listOrStrConstVal = (AsterixConstantValue) optFuncExpr.getConstantVal(0);
+ IAObject listOrStrObj = listOrStrConstVal.getObject();
+ ATypeTag typeTag = listOrStrObj.getType().getTypeTag();
+
+ if (!isEditDistanceFuncCompatible(typeTag, index.getIndexType())) {
+ return false;
+ }
+
+ AsterixConstantValue intConstVal = (AsterixConstantValue) optFuncExpr.getConstantVal(1);
+ IAObject intObj = intConstVal.getObject();
+ AInt32 edThresh = (AInt32) intObj;
+ int mergeThreshold = 0;
+
+ if (typeTag == ATypeTag.STRING) {
+ AString astr = (AString) listOrStrObj;
+ // Compute merge threshold depending on the query grams contain pre- and postfixing
+ if (optFuncExpr.containsPartialField()) {
+ mergeThreshold = (astr.getStringValue().length() - index.getGramLength() + 1)
+ - edThresh.getIntegerValue() * index.getGramLength();
} else {
- nonConstArg = arg2;
+ mergeThreshold = (astr.getStringValue().length() + index.getGramLength() - 1)
+ - edThresh.getIntegerValue() * index.getGramLength();
}
- if (nonConstArg.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- AbstractFunctionCallExpression nonConstfuncExpr = (AbstractFunctionCallExpression) nonConstArg;
- // We can use this index if the tokenization function matches the index type.
- if (nonConstfuncExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.WORD_TOKENS
- && (index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX)) {
- return true;
- }
- if (nonConstfuncExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.GRAM_TOKENS
- && (index.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)) {
- return true;
- }
+ }
+
+ if ((typeTag == ATypeTag.ORDEREDLIST)
+ && (index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX)) {
+ IACollection alist = (IACollection) listOrStrObj;
+ // Compute merge threshold.
+ mergeThreshold = alist.size() - edThresh.getIntegerValue();
+ }
+ if (mergeThreshold <= 0) {
+ // We cannot use index to optimize expr.
+ return false;
+ }
+ return true;
+ }
+
+ private boolean isJaccardFuncOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
+ //TODO we need to split join and select cases in order to check join case more thoroughly.
+
+ int variableCount = optFuncExpr.getNumLogicalVars();
+
+ //check whether gram-tokens function is optimizable
+ ScalarFunctionCallExpression funcExpr = null;
+ for (int i = 0; i < variableCount; i++) {
+ funcExpr = findTokensFunc(AsterixBuiltinFunctions.GRAM_TOKENS, optFuncExpr, i);
+ if (funcExpr != null) {
+ return isJaccardFuncCompatible(funcExpr, optFuncExpr.getTypeTag(i), index.getIndexType());
}
- // The non-constant arg is not a function call. Perhaps a variable?
- // We must have already verified during our analysis of the select condition, that this variable
- // refers to a list, or to a tokenization function.
- if (nonConstArg.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ }
+
+ //check whether word-tokens function is optimizable
+ for (int i = 0; i < variableCount; i++) {
+ funcExpr = findTokensFunc(AsterixBuiltinFunctions.WORD_TOKENS, optFuncExpr, i);
+ if (funcExpr != null) {
+ return isJaccardFuncCompatible(funcExpr, optFuncExpr.getTypeTag(i), index.getIndexType());
+ }
+ }
+
+ //check whether a search variable is optimizable
+ OptimizableOperatorSubTree subTree = null;
+ LogicalVariable targetVar = null;
+ for (int i = 0; i < variableCount; i++) {
+ subTree = optFuncExpr.getOperatorSubTree(i);
+ if (subTree == null)
+ continue;
+ targetVar = optFuncExpr.getLogicalVar(i);
+ if (targetVar == null)
+ continue;
+ return isJaccardFuncCompatible(optFuncExpr.getFuncExpr().getArguments().get(i).getValue(),
+ optFuncExpr.getTypeTag(i), index.getIndexType());
+ }
+
+ return false;
+ }
+
+ private ScalarFunctionCallExpression findTokensFunc(FunctionIdentifier funcId, IOptimizableFuncExpr optFuncExpr,
+ int subTreeIndex) {
+ //find either a gram-tokens or a word-tokens function that exists in optFuncExpr.subTrees' assignsAndUnnests
+ OptimizableOperatorSubTree subTree = null;
+ LogicalVariable targetVar = null;
+
+ subTree = optFuncExpr.getOperatorSubTree(subTreeIndex);
+ if (subTree == null) {
+ return null;
+ }
+
+ targetVar = optFuncExpr.getLogicalVar(subTreeIndex);
+ if (targetVar == null) {
+ return null;
+ }
+
+ for (AbstractLogicalOperator op : subTree.assignsAndUnnests) {
+ if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN)
+ continue;
+ List<Mutable<ILogicalExpression>> exprList = ((AssignOperator) op).getExpressions();
+ for (Mutable<ILogicalExpression> expr : exprList) {
+ if (expr.getValue().getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL)
+ continue;
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr.getValue();
+ if (funcExpr.getFunctionIdentifier() != funcId)
+ continue;
+ ILogicalExpression varExpr = funcExpr.getArguments().get(0).getValue();
+ if (varExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE)
+ continue;
+ if (((VariableReferenceExpression) varExpr).getVariableReference() == targetVar)
+ continue;
+ return (ScalarFunctionCallExpression) funcExpr;
+ }
+ }
+ return null;
+ }
+
+ private boolean isJaccardFuncCompatible(ILogicalExpression nonConstArg, ATypeTag typeTag, IndexType indexType) {
+ if (nonConstArg.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression nonConstfuncExpr = (AbstractFunctionCallExpression) nonConstArg;
+ // We can use this index if the tokenization function matches the index type.
+ if (nonConstfuncExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.WORD_TOKENS
+ && (indexType == IndexType.SINGLE_PARTITION_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX)) {
+ return true;
+ }
+ if (nonConstfuncExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.GRAM_TOKENS
+ && (indexType == IndexType.SINGLE_PARTITION_NGRAM_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)) {
return true;
}
}
- // We can only optimize contains with ngram indexes.
- if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.CONTAINS
- && (index.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)) {
- // Check that the constant search string has at least gramLength characters.
- AsterixConstantValue strConstVal = (AsterixConstantValue) optFuncExpr.getConstantVal(0);
- IAObject strObj = strConstVal.getObject();
- if (strObj.getType().getTypeTag() == ATypeTag.STRING) {
- AString astr = (AString) strObj;
- if (astr.getStringValue().length() >= index.getGramLength()) {
- return true;
- }
+
+ if (nonConstArg.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ if ((typeTag == ATypeTag.ORDEREDLIST || typeTag == ATypeTag.UNORDEREDLIST)
+ && (indexType == IndexType.SINGLE_PARTITION_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX)) {
+ return true;
}
+ // We assume that the given list variable doesn't have ngram list in it since it is unrealistic.
+ }
+ return false;
+ }
+
+ private boolean isContainsFuncOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
+ if (optFuncExpr.getNumLogicalVars() == 2) {
+ return isContainsFuncJoinOptimizable(index, optFuncExpr);
+ } else {
+ return isContainsFuncSelectOptimizable(index, optFuncExpr);
+ }
+ }
+
+ private boolean isContainsFuncSelectOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
+ AsterixConstantValue strConstVal = (AsterixConstantValue) optFuncExpr.getConstantVal(0);
+ IAObject strObj = strConstVal.getObject();
+ ATypeTag typeTag = strObj.getType().getTypeTag();
+
+ if (!isContainsFuncCompatible(typeTag, index.getIndexType())) {
+ return false;
+ }
+
+ // Check that the constant search string has at least gramLength characters.
+ if (strObj.getType().getTypeTag() == ATypeTag.STRING) {
+ AString astr = (AString) strObj;
+ if (astr.getStringValue().length() >= index.getGramLength()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean isContainsFuncJoinOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
+ return isContainsFuncCompatible(optFuncExpr.getTypeTag(0), index.getIndexType());
+ }
+
+ private boolean isContainsFuncCompatible(ATypeTag typeTag, IndexType indexType) {
+ //We can only optimize contains with ngram indexes.
+ if ((typeTag == ATypeTag.STRING)
+ && (indexType == IndexType.SINGLE_PARTITION_NGRAM_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)) {
+ return true;
}
return false;
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexJobGenParams.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexJobGenParams.java
index 40ab655..5808a83 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexJobGenParams.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexJobGenParams.java
@@ -45,8 +45,8 @@
}
public InvertedIndexJobGenParams(String indexName, IndexType indexType, String dataverseName, String datasetName,
- boolean retainInput, boolean requiresBroadcast) {
- super(indexName, indexType, dataverseName, datasetName, retainInput, requiresBroadcast);
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast) {
+ super(indexName, indexType, dataverseName, datasetName, retainInput, retainNull, requiresBroadcast);
}
public void setSearchModifierType(SearchModifierType searchModifierType) {
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java
index 79ccfc0..179ab83 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java
@@ -15,6 +15,7 @@
package edu.uci.ics.asterix.optimizer.rules.am;
import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
@@ -27,6 +28,7 @@
protected final AbstractFunctionCallExpression funcExpr;
protected final LogicalVariable[] logicalVars;
protected final String[] fieldNames;
+ protected final ATypeTag[] typeTags;
protected final OptimizableOperatorSubTree[] subTrees;
protected final IAlgebricksConstantValue[] constantVals;
protected boolean partialField;
@@ -37,7 +39,9 @@
this.logicalVars = logicalVars;
this.constantVals = constantVals;
this.fieldNames = new String[logicalVars.length];
+ this.typeTags = new ATypeTag[logicalVars.length];
this.subTrees = new OptimizableOperatorSubTree[logicalVars.length];
+
if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) {
this.partialField = true;
} else {
@@ -52,6 +56,7 @@
this.logicalVars = new LogicalVariable[] { logicalVar };
this.constantVals = new IAlgebricksConstantValue[] { constantVal };
this.fieldNames = new String[logicalVars.length];
+ this.typeTags = new ATypeTag[logicalVars.length];
this.subTrees = new OptimizableOperatorSubTree[logicalVars.length];
if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) {
this.partialField = true;
@@ -141,9 +146,19 @@
public void setPartialField(boolean partialField) {
this.partialField = partialField;
}
-
+
@Override
public boolean containsPartialField() {
return partialField;
}
+
+ @Override
+ public void setTypeTag(int index, ATypeTag typeTag) {
+ typeTags[index] = typeTag;
+ }
+
+ @Override
+ public ATypeTag getTypeTag(int index) {
+ return typeTags[index];
+ }
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 7110fbb..9103178 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -42,8 +42,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
@@ -67,8 +67,8 @@
}
@Override
- public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr, List<AbstractLogicalOperator> assignsAndUnnests,
- AccessMethodAnalysisContext analysisCtx) {
+ public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr,
+ List<AbstractLogicalOperator> assignsAndUnnests, AccessMethodAnalysisContext analysisCtx) {
boolean matches = AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx);
if (!matches) {
matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVars(funcExpr, analysisCtx);
@@ -93,7 +93,7 @@
// TODO: We can probably do something smarter here based on selectivity or MBR area.
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr,
- false, false, context);
+ analysisCtx, false, false, false, context);
if (primaryIndexUnnestOp == null) {
return false;
}
@@ -105,13 +105,15 @@
@Override
public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef,
OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException {
+ AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin)
+ throws AlgebricksException {
// Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex);
// Determine probe and index subtrees based on chosen index.
OptimizableOperatorSubTree indexSubTree = null;
OptimizableOperatorSubTree probeSubTree = null;
- if (leftSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(leftSubTree.dataset.getDatasetName())) {
+ if (!isLeftOuterJoin && leftSubTree.hasDataSourceScan()
+ && dataset.getDatasetName().equals(leftSubTree.dataset.getDatasetName())) {
indexSubTree = leftSubTree;
probeSubTree = rightSubTree;
} else if (rightSubTree.hasDataSourceScan()
@@ -119,17 +121,35 @@
indexSubTree = rightSubTree;
probeSubTree = leftSubTree;
}
+ if (indexSubTree == null) {
+ //This may happen for left outer join case
+ return false;
+ }
+
+ LogicalVariable newNullPlaceHolderVar = null;
+ if (isLeftOuterJoin) {
+ //get a new null place holder variable that is the first field variable of the primary key
+ //from the indexSubTree's datasourceScanOp
+ newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
+ }
+
// TODO: We can probably do something smarter here based on selectivity or MBR area.
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex,
- optFuncExpr, true, true, context);
+ optFuncExpr, analysisCtx, true, isLeftOuterJoin, true, context);
if (primaryIndexUnnestOp == null) {
return false;
}
+
+ if (isLeftOuterJoin) {
+ //reset the null place holder variable
+ AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+ }
+
indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp);
// Change join into a select with the same condition.
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
- SelectOperator topSelect = new SelectOperator(joinOp.getCondition());
+ SelectOperator topSelect = new SelectOperator(joinOp.getCondition(), isLeftOuterJoin, newNullPlaceHolderVar);
topSelect.getInputs().add(indexSubTree.rootRef);
topSelect.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(topSelect);
@@ -140,7 +160,8 @@
private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex, IOptimizableFuncExpr optFuncExpr,
- boolean retainInput, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
+ AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull,
+ boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
@@ -152,7 +173,7 @@
// we made sure indexSubTree has datasource scan
DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.dataSourceRef.getValue();
RTreeJobGenParams jobGenParams = new RTreeJobGenParams(chosenIndex.getIndexName(), IndexType.RTREE,
- dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
+ dataset.getDataverseName(), dataset.getDatasetName(), retainInput, retainNull, requiresBroadcast);
// A spatial object is serialized in the constant of the func expr we are optimizing.
// The R-Tree expects as input an MBR represented with 1 field per dimension.
// Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the R-Tree expects them).
@@ -197,9 +218,10 @@
UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput);
+
// Generate the rest of the upstream plan which feeds the search results into the primary index.
UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset,
- recordType, secondaryIndexUnnestOp, context, true, retainInput, false);
+ recordType, secondaryIndexUnnestOp, context, true, retainInput, false, false);
return primaryIndexUnnestOp;
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeJobGenParams.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeJobGenParams.java
index 114f250..b09e6ff 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeJobGenParams.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeJobGenParams.java
@@ -35,8 +35,8 @@
}
public RTreeJobGenParams(String indexName, IndexType indexType, String dataverseName, String datasetName,
- boolean retainInput, boolean requiresBroadcast) {
- super(indexName, indexType, dataverseName, datasetName, retainInput, requiresBroadcast);
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast) {
+ super(indexName, indexType, dataverseName, datasetName, retainInput, retainNull, requiresBroadcast);
}
public void writeToFuncArgs(List<Mutable<ILogicalExpression>> funcArgs) {
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
index c52f4d5..a2ecdc8 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -638,14 +638,14 @@
Pair<ILogicalOperator, LogicalVariable> pThen = ifexpr.getThenExpr().accept(this, nestedSource);
SelectOperator sel1 = new SelectOperator(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
- varCond)));
+ varCond)), false, null);
sel1.getInputs().add(new MutableObject<ILogicalOperator>(pThen.first));
Pair<ILogicalOperator, LogicalVariable> pElse = ifexpr.getElseExpr().accept(this, nestedSource);
AbstractFunctionCallExpression notVarCond = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), new MutableObject<ILogicalExpression>(
new VariableReferenceExpression(varCond)));
- SelectOperator sel2 = new SelectOperator(new MutableObject<ILogicalExpression>(notVarCond));
+ SelectOperator sel2 = new SelectOperator(new MutableObject<ILogicalExpression>(notVarCond), false, null);
sel2.getInputs().add(new MutableObject<ILogicalOperator>(pElse.first));
ILogicalPlan p1 = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(sel1));
@@ -824,7 +824,7 @@
AggregateFunctionCallExpression fAgg;
SelectOperator s;
if (qe.getQuantifier() == Quantifier.SOME) {
- s = new SelectOperator(new MutableObject<ILogicalExpression>(eo2.first));
+ s = new SelectOperator(new MutableObject<ILogicalExpression>(eo2.first), false, null);
s.getInputs().add(eo2.second);
fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.NON_EMPTY_STREAM,
new ArrayList<Mutable<ILogicalExpression>>());
@@ -832,7 +832,7 @@
List<Mutable<ILogicalExpression>> satExprList = new ArrayList<Mutable<ILogicalExpression>>(1);
satExprList.add(new MutableObject<ILogicalExpression>(eo2.first));
s = new SelectOperator(new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), satExprList)));
+ FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), satExprList)), false, null);
s.getInputs().add(eo2.second);
fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.EMPTY_STREAM,
new ArrayList<Mutable<ILogicalExpression>>());
@@ -922,7 +922,7 @@
public Pair<ILogicalOperator, LogicalVariable> visitWhereClause(WhereClause w, Mutable<ILogicalOperator> tupSource)
throws AsterixException {
Pair<ILogicalExpression, Mutable<ILogicalOperator>> p = aqlExprToAlgExpression(w.getWhereExpr(), tupSource);
- SelectOperator s = new SelectOperator(new MutableObject<ILogicalExpression>(p.first));
+ SelectOperator s = new SelectOperator(new MutableObject<ILogicalExpression>(p.first), false, null);
s.getInputs().add(p.second);
return new Pair<ILogicalOperator, LogicalVariable>(s, null);
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
index 0c7ff7d..bbd5c42 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
@@ -588,14 +588,14 @@
Pair<ILogicalOperator, LogicalVariable> pThen = ifexpr.getThenExpr().accept(this, nestedSource);
SelectOperator sel1 = new SelectOperator(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
- varCond)));
+ varCond)), false, null);
sel1.getInputs().add(new MutableObject<ILogicalOperator>(pThen.first));
Pair<ILogicalOperator, LogicalVariable> pElse = ifexpr.getElseExpr().accept(this, nestedSource);
AbstractFunctionCallExpression notVarCond = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), new MutableObject<ILogicalExpression>(
new VariableReferenceExpression(varCond)));
- SelectOperator sel2 = new SelectOperator(new MutableObject<ILogicalExpression>(notVarCond));
+ SelectOperator sel2 = new SelectOperator(new MutableObject<ILogicalExpression>(notVarCond), false, null);
sel2.getInputs().add(new MutableObject<ILogicalOperator>(pElse.first));
ILogicalPlan p1 = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(sel1));
@@ -766,7 +766,7 @@
AggregateFunctionCallExpression fAgg;
SelectOperator s;
if (qe.getQuantifier() == Quantifier.SOME) {
- s = new SelectOperator(new MutableObject<ILogicalExpression>(eo2.first));
+ s = new SelectOperator(new MutableObject<ILogicalExpression>(eo2.first), false, null);
s.getInputs().add(eo2.second);
fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.NON_EMPTY_STREAM,
new ArrayList<Mutable<ILogicalExpression>>());
@@ -774,7 +774,7 @@
List<Mutable<ILogicalExpression>> satExprList = new ArrayList<Mutable<ILogicalExpression>>(1);
satExprList.add(new MutableObject<ILogicalExpression>(eo2.first));
s = new SelectOperator(new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), satExprList)));
+ FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), satExprList)), false, null);
s.getInputs().add(eo2.second);
fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.EMPTY_STREAM,
new ArrayList<Mutable<ILogicalExpression>>());
@@ -864,7 +864,7 @@
public Pair<ILogicalOperator, LogicalVariable> visitWhereClause(WhereClause w, Mutable<ILogicalOperator> tupSource)
throws AsterixException {
Pair<ILogicalExpression, Mutable<ILogicalOperator>> p = aqlExprToAlgExpression(w.getWhereExpr(), tupSource);
- SelectOperator s = new SelectOperator(new MutableObject<ILogicalExpression>(p.first));
+ SelectOperator s = new SelectOperator(new MutableObject<ILogicalExpression>(p.first), false, null);
s.getInputs().add(p.second);
return new Pair<ILogicalOperator, LogicalVariable>(s, null);
diff --git a/asterix-app/data/twitter/tw_for_indexleftouterjoin.adm b/asterix-app/data/twitter/tw_for_indexleftouterjoin.adm
new file mode 100644
index 0000000..405dd08
--- /dev/null
+++ b/asterix-app/data/twitter/tw_for_indexleftouterjoin.adm
@@ -0,0 +1,250 @@
+{ "tweetid": 1i64, "user": { "screen-name": "WardLoewentsein@340", "lang": "en", "friends-count": 11, "statuses-count": 388, "name": "Ward Loewentsein", "followers-count": 129 }, "sender-location": point("42.83,72.44"), "send-time": datetime("2009-10-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile the speed is mind-blowing:)", "countA": 1, "countB": 26 }
+{ "tweetid": 2i64, "user": { "screen-name": "KyleGraham_120", "lang": "en", "friends-count": 55, "statuses-count": 231, "name": "Kyle Graham", "followers-count": 42 }, "sender-location": point("34.81,72.44"), "send-time": datetime("2011-09-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " hate samsung the 3G is horrible", "countA": 2, "countB": 131 }
+{ "tweetid": 3i64, "user": { "screen-name": "TateGarneys@542", "lang": "en", "friends-count": 74, "statuses-count": 370, "name": "Tate Garneys", "followers-count": 111 }, "sender-location": point("24.54,82.66"), "send-time": datetime("2009-12-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " hate iphone the shortcut-menu is bad", "countA": 3, "countB": 187 }
+{ "tweetid": 4i64, "user": { "screen-name": "BuckFields@708", "lang": "en", "friends-count": 20, "statuses-count": 469, "name": "Buck Fields", "followers-count": 181 }, "sender-location": point("38.14,68.1"), "send-time": datetime("2008-10-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " dislike samsung the speed is OMG", "countA": 4, "countB": 52 }
+{ "tweetid": 5i64, "user": { "screen-name": "NoreenBaldwin_373", "lang": "en", "friends-count": 89, "statuses-count": 144, "name": "Noreen Baldwin", "followers-count": 187 }, "sender-location": point("35.4,68.89"), "send-time": datetime("2008-10-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " hate motorola its 3G is OMG:(", "countA": 5, "countB": 35 }
+{ "tweetid": 6i64, "user": { "screen-name": "IselaHatcher_237", "lang": "en", "friends-count": 85, "statuses-count": 333, "name": "Isela Hatcher", "followers-count": 148 }, "sender-location": point("42.75,78.5"), "send-time": datetime("2011-10-15T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " hate sprint the wireless is terrible:(", "countA": 6, "countB": 61 }
+{ "tweetid": 7i64, "user": { "screen-name": "NicolaJolce$660", "lang": "en", "friends-count": 45, "statuses-count": 420, "name": "Nicola Jolce", "followers-count": 12 }, "sender-location": point("48.16,71.59"), "send-time": datetime("2005-11-23T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-command" }}, "message-text": " like motorola its voice-command is amazing", "countA": 7, "countB": 47 }
+{ "tweetid": 8i64, "user": { "screen-name": "MorganKeppel_176", "lang": "en", "friends-count": 74, "statuses-count": 190, "name": "Morgan Keppel", "followers-count": 2 }, "sender-location": point("36.17,72.56"), "send-time": datetime("2011-12-02T10:10:00.000Z"), "referred-topics": {{ "verizon", "3G" }}, "message-text": " hate verizon the 3G is OMG:(", "countA": 8, "countB": 98 }
+{ "tweetid": 9i64, "user": { "screen-name": "GerardMcdonald$43", "lang": "en", "friends-count": 72, "statuses-count": 151, "name": "Gerard Mcdonald", "followers-count": 96 }, "sender-location": point("38.02,70.38"), "send-time": datetime("2005-10-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " love sprint its voice-clarity is amazing", "countA": 9, "countB": 69 }
+{ "tweetid": 10i64, "user": { "screen-name": "WynonnaButler_286", "lang": "en", "friends-count": 30, "statuses-count": 375, "name": "Wynonna Butler", "followers-count": 78 }, "sender-location": point("38.71,90.05"), "send-time": datetime("2008-09-21T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " love motorola its wireless is good:)", "countA": 10, "countB": 75 }
+{ "tweetid": 11i64, "user": { "screen-name": "BrodyKing@977", "lang": "en", "friends-count": 3, "statuses-count": 62, "name": "Brody King", "followers-count": 106 }, "sender-location": point("32.26,73.48"), "send-time": datetime("2007-05-20T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " can't stand sprint the shortcut-menu is OMG", "countA": 11, "countB": 28 }
+{ "tweetid": 12i64, "user": { "screen-name": "ValentineSchofield@448", "lang": "en", "friends-count": 16, "statuses-count": 260, "name": "Valentine Schofield", "followers-count": 136 }, "sender-location": point("24.99,70.66"), "send-time": datetime("2009-07-26T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " can't stand sprint its shortcut-menu is terrible", "countA": 12, "countB": 159 }
+{ "tweetid": 13i64, "user": { "screen-name": "MaryroseBennett#483", "lang": "en", "friends-count": 99, "statuses-count": 496, "name": "Maryrose Bennett", "followers-count": 57 }, "sender-location": point("38.94,93.31"), "send-time": datetime("2005-01-06T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " love at&t its platform is awesome", "countA": 13, "countB": 13 }
+{ "tweetid": 14i64, "user": { "screen-name": "GarlandAlliman$490", "lang": "en", "friends-count": 23, "statuses-count": 146, "name": "Garland Alliman", "followers-count": 51 }, "sender-location": point("29.96,75.0"), "send-time": datetime("2009-09-26T10:10:00.000Z"), "referred-topics": {{ "motorola", "platform" }}, "message-text": " dislike motorola the platform is bad:(", "countA": 14, "countB": 74 }
+{ "tweetid": 15i64, "user": { "screen-name": "AnnabelPirl$171", "lang": "en", "friends-count": 43, "statuses-count": 402, "name": "Annabel Pirl", "followers-count": 137 }, "sender-location": point("27.22,86.32"), "send-time": datetime("2008-09-08T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " like at&t the network is amazing:)", "countA": 15, "countB": 19 }
+{ "tweetid": 16i64, "user": { "screen-name": "JarvisPickering@42", "lang": "en", "friends-count": 98, "statuses-count": 498, "name": "Jarvis Pickering", "followers-count": 20 }, "sender-location": point("30.54,81.6"), "send-time": datetime("2005-11-05T10:10:00.000Z"), "referred-topics": {{ "iphone", "wireless" }}, "message-text": " love iphone the wireless is awesome", "countA": 16, "countB": 27 }
+{ "tweetid": 17i64, "user": { "screen-name": "LillyHoffhants@595", "lang": "en", "friends-count": 35, "statuses-count": 391, "name": "Lilly Hoffhants", "followers-count": 129 }, "sender-location": point("47.96,95.21"), "send-time": datetime("2007-02-14T10:10:00.000Z"), "referred-topics": {{ "verizon", "customer-service" }}, "message-text": " like verizon its customer-service is amazing", "countA": 17, "countB": 55 }
+{ "tweetid": 18i64, "user": { "screen-name": "AllanPolson_455", "lang": "en", "friends-count": 36, "statuses-count": 227, "name": "Allan Polson", "followers-count": 113 }, "sender-location": point("46.39,73.85"), "send-time": datetime("2009-12-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " hate iphone the platform is bad", "countA": 18, "countB": 199 }
+{ "tweetid": 19i64, "user": { "screen-name": "DonnieWentzel#857", "lang": "en", "friends-count": 76, "statuses-count": 23, "name": "Donnie Wentzel", "followers-count": 78 }, "sender-location": point("37.21,95.76"), "send-time": datetime("2012-04-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "reachability" }}, "message-text": " like iphone the reachability is mind-blowing", "countA": 19, "countB": 68 }
+{ "tweetid": 20i64, "user": { "screen-name": "TraversFaast@428", "lang": "en", "friends-count": 42, "statuses-count": 70, "name": "Travers Faast", "followers-count": 116 }, "sender-location": point("26.54,74.71"), "send-time": datetime("2010-06-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone the shortcut-menu is mind-blowing", "countA": 20, "countB": 18 }
+{ "tweetid": 21i64, "user": { "screen-name": "KameronSandford#555", "lang": "en", "friends-count": 22, "statuses-count": 104, "name": "Kameron Sandford", "followers-count": 17 }, "sender-location": point("25.8,88.76"), "send-time": datetime("2007-11-20T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " can't stand at&t the network is bad:(", "countA": 21, "countB": 64 }
+{ "tweetid": 22i64, "user": { "screen-name": "TiannaArmitage_372", "lang": "en", "friends-count": 32, "statuses-count": 307, "name": "Tianna Armitage", "followers-count": 126 }, "sender-location": point("45.79,80.75"), "send-time": datetime("2005-04-14T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola its reachability is terrible:(", "countA": 22, "countB": 126 }
+{ "tweetid": 23i64, "user": { "screen-name": "NevadaCattley#858", "lang": "en", "friends-count": 74, "statuses-count": 389, "name": "Nevada Cattley", "followers-count": 17 }, "sender-location": point("43.6,93.24"), "send-time": datetime("2007-03-11T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " hate at&t the network is horrible", "countA": 23, "countB": 74 }
+{ "tweetid": 24i64, "user": { "screen-name": "ReannonEisenhart#637", "lang": "en", "friends-count": 34, "statuses-count": 235, "name": "Reannon Eisenhart", "followers-count": 145 }, "sender-location": point("47.56,76.49"), "send-time": datetime("2012-03-20T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-clarity" }}, "message-text": " can't stand samsung its voice-clarity is terrible:(", "countA": 24, "countB": 70 }
+{ "tweetid": 25i64, "user": { "screen-name": "LillyMang#928", "lang": "en", "friends-count": 20, "statuses-count": 40, "name": "Lilly Mang", "followers-count": 47 }, "sender-location": point("38.68,94.93"), "send-time": datetime("2006-04-21T10:10:00.000Z"), "referred-topics": {{ "sprint", "customization" }}, "message-text": " love sprint its customization is awesome", "countA": 25, "countB": 86 }
+{ "tweetid": 26i64, "user": { "screen-name": "MicaBusk$903", "lang": "en", "friends-count": 87, "statuses-count": 164, "name": "Mica Busk", "followers-count": 92 }, "sender-location": point("45.47,90.97"), "send-time": datetime("2008-01-24T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " hate motorola the touch-screen is terrible", "countA": 26, "countB": 11 }
+{ "tweetid": 27i64, "user": { "screen-name": "PiaHildyard_915", "lang": "en", "friends-count": 92, "statuses-count": 302, "name": "Pia Hildyard", "followers-count": 16 }, "sender-location": point("43.76,68.58"), "send-time": datetime("2007-08-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " dislike iphone its platform is bad:(", "countA": 27, "countB": 125 }
+{ "tweetid": 28i64, "user": { "screen-name": "CamelliaSiegrist_676", "lang": "en", "friends-count": 73, "statuses-count": 392, "name": "Camellia Siegrist", "followers-count": 193 }, "sender-location": point("24.94,77.95"), "send-time": datetime("2007-12-26T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-command" }}, "message-text": " can't stand verizon its voice-command is bad", "countA": 28, "countB": 123 }
+{ "tweetid": 29i64, "user": { "screen-name": "BurtTaggart_922", "lang": "en", "friends-count": 49, "statuses-count": 62, "name": "Burt Taggart", "followers-count": 134 }, "sender-location": point("35.67,97.43"), "send-time": datetime("2011-07-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is awesome", "countA": 29, "countB": 115 }
+{ "tweetid": 30i64, "user": { "screen-name": "MarlaHill@215", "lang": "en", "friends-count": 84, "statuses-count": 305, "name": "Marla Hill", "followers-count": 71 }, "sender-location": point("24.29,84.28"), "send-time": datetime("2012-07-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "3G" }}, "message-text": " like t-mobile the 3G is awesome", "countA": 30, "countB": 36 }
+{ "tweetid": 31i64, "user": { "screen-name": "CaitlynChristman@452", "lang": "en", "friends-count": 57, "statuses-count": 414, "name": "Caitlyn Christman", "followers-count": 67 }, "sender-location": point("41.04,85.13"), "send-time": datetime("2005-11-05T10:10:00.000Z"), "referred-topics": {{ "samsung", "voicemail-service" }}, "message-text": " love samsung the voicemail-service is mind-blowing:)", "countA": 31, "countB": 59 }
+{ "tweetid": 32i64, "user": { "screen-name": "BraxtonBonner#527", "lang": "en", "friends-count": 21, "statuses-count": 427, "name": "Braxton Bonner", "followers-count": 168 }, "sender-location": point("34.25,86.09"), "send-time": datetime("2011-10-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " dislike motorola its 3G is horrible:(", "countA": 32, "countB": 38 }
+{ "tweetid": 33i64, "user": { "screen-name": "WilmaSouthern@238", "lang": "en", "friends-count": 83, "statuses-count": 413, "name": "Wilma Southern", "followers-count": 24 }, "sender-location": point("34.71,69.57"), "send-time": datetime("2005-02-19T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " dislike sprint the wireless is OMG:(", "countA": 33, "countB": 105 }
+{ "tweetid": 34i64, "user": { "screen-name": "MaxNash$802", "lang": "en", "friends-count": 13, "statuses-count": 189, "name": "Max Nash", "followers-count": 39 }, "sender-location": point("48.12,89.23"), "send-time": datetime("2012-02-17T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " dislike motorola its touch-screen is horrible", "countA": 34, "countB": 185 }
+{ "tweetid": 35i64, "user": { "screen-name": "HannahWarrick_843", "lang": "en", "friends-count": 14, "statuses-count": 10, "name": "Hannah Warrick", "followers-count": 2 }, "sender-location": point("32.75,69.94"), "send-time": datetime("2007-09-14T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "plan" }}, "message-text": " like t-mobile the plan is amazing:)", "countA": 35, "countB": 176 }
+{ "tweetid": 36i64, "user": { "screen-name": "SherikaBarth#732", "lang": "en", "friends-count": 80, "statuses-count": 277, "name": "Sherika Barth", "followers-count": 138 }, "sender-location": point("34.85,66.87"), "send-time": datetime("2011-10-13T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " can't stand at&t the plan is OMG", "countA": 36, "countB": 147 }
+{ "tweetid": 37i64, "user": { "screen-name": "SabinaCattley$355", "lang": "en", "friends-count": 67, "statuses-count": 20, "name": "Sabina Cattley", "followers-count": 104 }, "sender-location": point("40.22,71.18"), "send-time": datetime("2007-04-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-clarity" }}, "message-text": " can't stand samsung its voice-clarity is bad:(", "countA": 37, "countB": 55 }
+{ "tweetid": 38i64, "user": { "screen-name": "KimberlyVeith$848", "lang": "en", "friends-count": 43, "statuses-count": 274, "name": "Kimberly Veith", "followers-count": 14 }, "sender-location": point("34.94,83.17"), "send-time": datetime("2011-07-15T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " hate samsung the speed is horrible:(", "countA": 38, "countB": 89 }
+{ "tweetid": 39i64, "user": { "screen-name": "AdrianneMackendoerfer_478", "lang": "en", "friends-count": 2, "statuses-count": 125, "name": "Adrianne Mackendoerfer", "followers-count": 113 }, "sender-location": point("40.14,78.49"), "send-time": datetime("2010-10-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-command" }}, "message-text": " hate motorola its voice-command is OMG", "countA": 39, "countB": 97 }
+{ "tweetid": 40i64, "user": { "screen-name": "MunroWire@995", "lang": "en", "friends-count": 89, "statuses-count": 336, "name": "Munro Wire", "followers-count": 181 }, "sender-location": point("30.94,80.83"), "send-time": datetime("2009-05-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " love verizon the network is good", "countA": 40, "countB": 193 }
+{ "tweetid": 41i64, "user": { "screen-name": "AmadaAft@648", "lang": "en", "friends-count": 50, "statuses-count": 127, "name": "Amada Aft", "followers-count": 20 }, "sender-location": point("32.88,81.46"), "send-time": datetime("2010-04-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " can't stand iphone its customer-service is OMG:(", "countA": 41, "countB": 169 }
+{ "tweetid": 42i64, "user": { "screen-name": "SalenaMcfall_717", "lang": "en", "friends-count": 30, "statuses-count": 93, "name": "Salena Mcfall", "followers-count": 184 }, "sender-location": point("47.86,71.93"), "send-time": datetime("2010-02-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "network" }}, "message-text": " can't stand t-mobile its network is bad", "countA": 42, "countB": 130 }
+{ "tweetid": 43i64, "user": { "screen-name": "JeniferCanham$317", "lang": "en", "friends-count": 63, "statuses-count": 344, "name": "Jenifer Canham", "followers-count": 132 }, "sender-location": point("25.68,81.87"), "send-time": datetime("2010-04-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " hate motorola the 3G is OMG:(", "countA": 43, "countB": 153 }
+{ "tweetid": 44i64, "user": { "screen-name": "NannieBender$656", "lang": "en", "friends-count": 26, "statuses-count": 84, "name": "Nannie Bender", "followers-count": 184 }, "sender-location": point("47.46,85.04"), "send-time": datetime("2007-06-08T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " dislike samsung its voice-command is bad", "countA": 44, "countB": 12 }
+{ "tweetid": 45i64, "user": { "screen-name": "ThaoKooser@875", "lang": "en", "friends-count": 60, "statuses-count": 289, "name": "Thao Kooser", "followers-count": 8 }, "sender-location": point("37.02,87.94"), "send-time": datetime("2005-11-28T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " like verizon its network is amazing:)", "countA": 45, "countB": 151 }
+{ "tweetid": 46i64, "user": { "screen-name": "AugustaBaumgartner_385", "lang": "en", "friends-count": 17, "statuses-count": 70, "name": "Augusta Baumgartner", "followers-count": 162 }, "sender-location": point("24.83,73.16"), "send-time": datetime("2008-09-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " like verizon its network is mind-blowing", "countA": 46, "countB": 37 }
+{ "tweetid": 47i64, "user": { "screen-name": "OtisHill_124", "lang": "en", "friends-count": 46, "statuses-count": 68, "name": "Otis Hill", "followers-count": 29 }, "sender-location": point("36.01,86.76"), "send-time": datetime("2011-05-16T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " dislike t-mobile the touch-screen is horrible:(", "countA": 47, "countB": 27 }
+{ "tweetid": 48i64, "user": { "screen-name": "ZolaJudge@572", "lang": "en", "friends-count": 8, "statuses-count": 39, "name": "Zola Judge", "followers-count": 36 }, "sender-location": point("42.67,91.8"), "send-time": datetime("2009-03-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " like motorola the network is awesome", "countA": 48, "countB": 16 }
+{ "tweetid": 49i64, "user": { "screen-name": "ChristianaWeisgarber$35", "lang": "en", "friends-count": 40, "statuses-count": 427, "name": "Christiana Weisgarber", "followers-count": 20 }, "sender-location": point("36.91,86.0"), "send-time": datetime("2010-04-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " hate sprint its plan is terrible", "countA": 49, "countB": 28 }
+{ "tweetid": 50i64, "user": { "screen-name": "MollyGarneis_210", "lang": "en", "friends-count": 6, "statuses-count": 453, "name": "Molly Garneis", "followers-count": 185 }, "sender-location": point("44.42,87.86"), "send-time": datetime("2012-04-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " love t-mobile its wireless is good:)", "countA": 50, "countB": 123 }
+{ "tweetid": 51i64, "user": { "screen-name": "HarlanHanseu#420", "lang": "en", "friends-count": 66, "statuses-count": 295, "name": "Harlan Hanseu", "followers-count": 99 }, "sender-location": point("37.65,70.54"), "send-time": datetime("2006-05-11T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " can't stand samsung the speed is terrible:(", "countA": 51, "countB": 94 }
+{ "tweetid": 52i64, "user": { "screen-name": "DelorseSloan#229", "lang": "en", "friends-count": 84, "statuses-count": 287, "name": "Delorse Sloan", "followers-count": 20 }, "sender-location": point("27.12,78.69"), "send-time": datetime("2011-02-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " dislike motorola its touch-screen is horrible:(", "countA": 52, "countB": 156 }
+{ "tweetid": 53i64, "user": { "screen-name": "MylesEwing@54", "lang": "en", "friends-count": 45, "statuses-count": 411, "name": "Myles Ewing", "followers-count": 23 }, "sender-location": point("25.82,97.9"), "send-time": datetime("2007-11-10T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " like at&t its network is mind-blowing:)", "countA": 53, "countB": 174 }
+{ "tweetid": 54i64, "user": { "screen-name": "OprahClark_160", "lang": "en", "friends-count": 26, "statuses-count": 299, "name": "Oprah Clark", "followers-count": 161 }, "sender-location": point("36.1,87.24"), "send-time": datetime("2010-04-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile its shortcut-menu is good", "countA": 54, "countB": 128 }
+{ "tweetid": 55i64, "user": { "screen-name": "CoreyRichards#130", "lang": "en", "friends-count": 45, "statuses-count": 420, "name": "Corey Richards", "followers-count": 102 }, "sender-location": point("42.77,72.16"), "send-time": datetime("2012-07-20T10:10:00.000Z"), "referred-topics": {{ "at&t", "customer-service" }}, "message-text": " can't stand at&t its customer-service is horrible", "countA": 55, "countB": 30 }
+{ "tweetid": 56i64, "user": { "screen-name": "GwendolenHahn#673", "lang": "en", "friends-count": 62, "statuses-count": 426, "name": "Gwendolen Hahn", "followers-count": 158 }, "sender-location": point("39.76,90.94"), "send-time": datetime("2009-05-10T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " dislike samsung its 3G is bad:(", "countA": 56, "countB": 50 }
+{ "tweetid": 57i64, "user": { "screen-name": "DewayneBallou@258", "lang": "en", "friends-count": 42, "statuses-count": 215, "name": "Dewayne Ballou", "followers-count": 85 }, "sender-location": point("28.45,75.02"), "send-time": datetime("2006-08-07T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile the speed is mind-blowing:)", "countA": 57, "countB": 61 }
+{ "tweetid": 58i64, "user": { "screen-name": "RenayReese@543", "lang": "en", "friends-count": 80, "statuses-count": 459, "name": "Renay Reese", "followers-count": 102 }, "sender-location": point("38.09,77.66"), "send-time": datetime("2007-12-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " love motorola the speed is awesome:)", "countA": 58, "countB": 35 }
+{ "tweetid": 59i64, "user": { "screen-name": "SamuelHoffhants@740", "lang": "en", "friends-count": 53, "statuses-count": 64, "name": "Samuel Hoffhants", "followers-count": 150 }, "sender-location": point("48.86,97.19"), "send-time": datetime("2009-11-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " dislike sprint its speed is horrible", "countA": 59, "countB": 95 }
+{ "tweetid": 60i64, "user": { "screen-name": "GarlandOneal@886", "lang": "en", "friends-count": 12, "statuses-count": 481, "name": "Garland Oneal", "followers-count": 13 }, "sender-location": point("37.6,75.74"), "send-time": datetime("2005-05-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint its touch-screen is OMG:(", "countA": 60, "countB": 2 }
+{ "tweetid": 61i64, "user": { "screen-name": "LukeHoopengarner@327", "lang": "en", "friends-count": 77, "statuses-count": 224, "name": "Luke Hoopengarner", "followers-count": 107 }, "sender-location": point("46.38,80.88"), "send-time": datetime("2006-11-13T10:10:00.000Z"), "referred-topics": {{ "sprint", "voicemail-service" }}, "message-text": " can't stand sprint its voicemail-service is bad:(", "countA": 61, "countB": 12 }
+{ "tweetid": 62i64, "user": { "screen-name": "AudieStahl@296", "lang": "en", "friends-count": 89, "statuses-count": 90, "name": "Audie Stahl", "followers-count": 74 }, "sender-location": point("32.48,96.01"), "send-time": datetime("2010-04-23T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t its platform is terrible:(", "countA": 62, "countB": 47 }
+{ "tweetid": 63i64, "user": { "screen-name": "ArielleErrett_963", "lang": "en", "friends-count": 15, "statuses-count": 385, "name": "Arielle Errett", "followers-count": 34 }, "sender-location": point("25.39,82.02"), "send-time": datetime("2012-02-20T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " love verizon its signal is awesome:)", "countA": 63, "countB": 149 }
+{ "tweetid": 64i64, "user": { "screen-name": "MiltonWeldi#571", "lang": "en", "friends-count": 72, "statuses-count": 236, "name": "Milton Weldi", "followers-count": 128 }, "sender-location": point("26.08,85.94"), "send-time": datetime("2007-09-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " like sprint the reachability is awesome", "countA": 64, "countB": 158 }
+{ "tweetid": 65i64, "user": { "screen-name": "CarlineAft_666", "lang": "en", "friends-count": 25, "statuses-count": 352, "name": "Carline Aft", "followers-count": 59 }, "sender-location": point("29.33,78.49"), "send-time": datetime("2012-01-18T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " can't stand samsung the customization is terrible:(", "countA": 65, "countB": 77 }
+{ "tweetid": 66i64, "user": { "screen-name": "TenaGronko#55", "lang": "en", "friends-count": 91, "statuses-count": 2, "name": "Tena Gronko", "followers-count": 19 }, "sender-location": point("40.14,73.21"), "send-time": datetime("2011-03-18T10:10:00.000Z"), "referred-topics": {{ "at&t", "voice-command" }}, "message-text": " hate at&t the voice-command is horrible", "countA": 66, "countB": 73 }
+{ "tweetid": 67i64, "user": { "screen-name": "DanialBrinigh#499", "lang": "en", "friends-count": 67, "statuses-count": 413, "name": "Danial Brinigh", "followers-count": 4 }, "sender-location": point("41.26,97.09"), "send-time": datetime("2008-02-28T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " can't stand motorola the 3G is OMG:(", "countA": 67, "countB": 47 }
+{ "tweetid": 68i64, "user": { "screen-name": "GretaBusk#270", "lang": "en", "friends-count": 30, "statuses-count": 150, "name": "Greta Busk", "followers-count": 124 }, "sender-location": point("30.35,86.51"), "send-time": datetime("2008-03-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola the reachability is bad:(", "countA": 68, "countB": 21 }
+{ "tweetid": 69i64, "user": { "screen-name": "DanielBurch@155", "lang": "en", "friends-count": 5, "statuses-count": 268, "name": "Daniel Burch", "followers-count": 178 }, "sender-location": point("45.31,66.89"), "send-time": datetime("2009-06-27T10:10:00.000Z"), "referred-topics": {{ "samsung", "network" }}, "message-text": " hate samsung its network is horrible", "countA": 69, "countB": 153 }
+{ "tweetid": 70i64, "user": { "screen-name": "JaynaBash@532", "lang": "en", "friends-count": 90, "statuses-count": 244, "name": "Jayna Bash", "followers-count": 184 }, "sender-location": point("43.92,69.28"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " can't stand samsung the platform is bad", "countA": 70, "countB": 133 }
+{ "tweetid": 71i64, "user": { "screen-name": "PatriciaCason#475", "lang": "en", "friends-count": 50, "statuses-count": 149, "name": "Patricia Cason", "followers-count": 114 }, "sender-location": point("43.74,69.29"), "send-time": datetime("2009-08-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is amazing:)", "countA": 71, "countB": 185 }
+{ "tweetid": 72i64, "user": { "screen-name": "KatharineElsas_215", "lang": "en", "friends-count": 69, "statuses-count": 128, "name": "Katharine Elsas", "followers-count": 114 }, "sender-location": point("29.05,94.41"), "send-time": datetime("2010-09-25T10:10:00.000Z"), "referred-topics": {{ "samsung", "signal" }}, "message-text": " can't stand samsung the signal is OMG:(", "countA": 72, "countB": 31 }
+{ "tweetid": 73i64, "user": { "screen-name": "YorkSanborn_951", "lang": "en", "friends-count": 69, "statuses-count": 375, "name": "York Sanborn", "followers-count": 15 }, "sender-location": point("43.92,94.49"), "send-time": datetime("2010-09-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " dislike motorola its reachability is bad:(", "countA": 73, "countB": 61 }
+{ "tweetid": 74i64, "user": { "screen-name": "AlbertoDull$598", "lang": "en", "friends-count": 29, "statuses-count": 181, "name": "Alberto Dull", "followers-count": 192 }, "sender-location": point("25.6,85.23"), "send-time": datetime("2005-09-22T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " dislike samsung the wireless is OMG:(", "countA": 74, "countB": 32 }
+{ "tweetid": 75i64, "user": { "screen-name": "EnriqueFaast$123", "lang": "en", "friends-count": 9, "statuses-count": 24, "name": "Enrique Faast", "followers-count": 24 }, "sender-location": point("30.09,72.93"), "send-time": datetime("2009-10-17T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voicemail-service" }}, "message-text": " dislike t-mobile its voicemail-service is horrible", "countA": 75, "countB": 185 }
+{ "tweetid": 76i64, "user": { "screen-name": "AndreaBruxner$43", "lang": "en", "friends-count": 37, "statuses-count": 279, "name": "Andrea Bruxner", "followers-count": 118 }, "sender-location": point("30.39,92.92"), "send-time": datetime("2011-04-18T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " like t-mobile its customization is awesome", "countA": 76, "countB": 146 }
+{ "tweetid": 77i64, "user": { "screen-name": "LashawnaKemble$318", "lang": "en", "friends-count": 53, "statuses-count": 44, "name": "Lashawna Kemble", "followers-count": 102 }, "sender-location": point("46.29,93.16"), "send-time": datetime("2010-12-09T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " dislike samsung the voice-command is horrible", "countA": 77, "countB": 0 }
+{ "tweetid": 78i64, "user": { "screen-name": "RodolfoWoodworth#419", "lang": "en", "friends-count": 2, "statuses-count": 82, "name": "Rodolfo Woodworth", "followers-count": 16 }, "sender-location": point("44.92,70.03"), "send-time": datetime("2008-12-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " like motorola the touch-screen is awesome", "countA": 78, "countB": 140 }
+{ "tweetid": 79i64, "user": { "screen-name": "AbramCourtney_384", "lang": "en", "friends-count": 10, "statuses-count": 33, "name": "Abram Courtney", "followers-count": 138 }, "sender-location": point("34.9,96.91"), "send-time": datetime("2007-03-15T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " hate at&t the plan is bad:(", "countA": 79, "countB": 193 }
+{ "tweetid": 80i64, "user": { "screen-name": "LaurindaRosensteel@202", "lang": "en", "friends-count": 19, "statuses-count": 222, "name": "Laurinda Rosensteel", "followers-count": 20 }, "sender-location": point("47.86,92.66"), "send-time": datetime("2008-09-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " hate samsung its platform is horrible:(", "countA": 80, "countB": 39 }
+{ "tweetid": 81i64, "user": { "screen-name": "JarrettBratton@573", "lang": "en", "friends-count": 90, "statuses-count": 287, "name": "Jarrett Bratton", "followers-count": 18 }, "sender-location": point("33.85,75.61"), "send-time": datetime("2006-05-06T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " like sprint its network is amazing:)", "countA": 81, "countB": 105 }
+{ "tweetid": 82i64, "user": { "screen-name": "EleanorBicknell$880", "lang": "en", "friends-count": 35, "statuses-count": 444, "name": "Eleanor Bicknell", "followers-count": 199 }, "sender-location": point("42.31,95.69"), "send-time": datetime("2008-04-19T10:10:00.000Z"), "referred-topics": {{ "samsung", "plan" }}, "message-text": " can't stand samsung its plan is terrible", "countA": 82, "countB": 140 }
+{ "tweetid": 83i64, "user": { "screen-name": "MannyWerner#209", "lang": "en", "friends-count": 85, "statuses-count": 476, "name": "Manny Werner", "followers-count": 78 }, "sender-location": point("35.76,80.06"), "send-time": datetime("2007-11-17T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " dislike samsung the platform is terrible", "countA": 83, "countB": 163 }
+{ "tweetid": 84i64, "user": { "screen-name": "TeshaReade_713", "lang": "en", "friends-count": 74, "statuses-count": 99, "name": "Tesha Reade", "followers-count": 112 }, "sender-location": point("26.69,82.05"), "send-time": datetime("2007-01-04T10:10:00.000Z"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon the shortcut-menu is mind-blowing", "countA": 84, "countB": 172 }
+{ "tweetid": 85i64, "user": { "screen-name": "WanCrissman$283", "lang": "en", "friends-count": 37, "statuses-count": 384, "name": "Wan Crissman", "followers-count": 176 }, "sender-location": point("42.3,69.18"), "send-time": datetime("2006-10-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " love at&t its network is awesome:)", "countA": 85, "countB": 77 }
+{ "tweetid": 86i64, "user": { "screen-name": "DeedeeMccallum#158", "lang": "en", "friends-count": 81, "statuses-count": 370, "name": "Deedee Mccallum", "followers-count": 104 }, "sender-location": point("46.35,92.36"), "send-time": datetime("2011-09-22T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-command" }}, "message-text": " like verizon the voice-command is amazing:)", "countA": 86, "countB": 156 }
+{ "tweetid": 87i64, "user": { "screen-name": "AileenAft@340", "lang": "en", "friends-count": 87, "statuses-count": 476, "name": "Aileen Aft", "followers-count": 14 }, "sender-location": point("41.44,95.97"), "send-time": datetime("2012-07-16T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " love motorola its network is mind-blowing", "countA": 87, "countB": 155 }
+{ "tweetid": 88i64, "user": { "screen-name": "BurtonLinton_390", "lang": "en", "friends-count": 13, "statuses-count": 462, "name": "Burton Linton", "followers-count": 34 }, "sender-location": point("45.22,88.29"), "send-time": datetime("2011-12-15T10:10:00.000Z"), "referred-topics": {{ "iphone", "reachability" }}, "message-text": " love iphone the reachability is awesome", "countA": 88, "countB": 95 }
+{ "tweetid": 89i64, "user": { "screen-name": "DamionJoghs_943", "lang": "en", "friends-count": 18, "statuses-count": 388, "name": "Damion Joghs", "followers-count": 111 }, "sender-location": point("36.32,83.38"), "send-time": datetime("2011-06-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " love motorola its touch-screen is awesome", "countA": 89, "countB": 85 }
+{ "tweetid": 90i64, "user": { "screen-name": "LatoshaCowart_858", "lang": "en", "friends-count": 14, "statuses-count": 318, "name": "Latosha Cowart", "followers-count": 27 }, "sender-location": point("26.63,82.77"), "send-time": datetime("2011-10-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " love motorola the plan is good:)", "countA": 90, "countB": 139 }
+{ "tweetid": 91i64, "user": { "screen-name": "LoganPowers$336", "lang": "en", "friends-count": 52, "statuses-count": 154, "name": "Logan Powers", "followers-count": 28 }, "sender-location": point("30.66,96.22"), "send-time": datetime("2011-02-25T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " hate verizon the reachability is OMG:(", "countA": 91, "countB": 198 }
+{ "tweetid": 92i64, "user": { "screen-name": "NeilParkinson#794", "lang": "en", "friends-count": 18, "statuses-count": 365, "name": "Neil Parkinson", "followers-count": 27 }, "sender-location": point("31.25,71.75"), "send-time": datetime("2009-12-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t the platform is terrible:(", "countA": 92, "countB": 59 }
+{ "tweetid": 93i64, "user": { "screen-name": "GoddardFiscina$655", "lang": "en", "friends-count": 10, "statuses-count": 388, "name": "Goddard Fiscina", "followers-count": 142 }, "sender-location": point("33.4,72.55"), "send-time": datetime("2009-04-15T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " like samsung the speed is mind-blowing", "countA": 93, "countB": 55 }
+{ "tweetid": 94i64, "user": { "screen-name": "JacindaCressman_698", "lang": "en", "friends-count": 50, "statuses-count": 380, "name": "Jacinda Cressman", "followers-count": 112 }, "sender-location": point("33.68,85.33"), "send-time": datetime("2010-09-07T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " like sprint its network is amazing:)", "countA": 94, "countB": 128 }
+{ "tweetid": 95i64, "user": { "screen-name": "NelsonWilks_476", "lang": "en", "friends-count": 43, "statuses-count": 249, "name": "Nelson Wilks", "followers-count": 47 }, "sender-location": point("26.2,74.63"), "send-time": datetime("2010-10-28T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " love iphone the shortcut-menu is mind-blowing", "countA": 95, "countB": 144 }
+{ "tweetid": 96i64, "user": { "screen-name": "FelipeBeach_761", "lang": "en", "friends-count": 70, "statuses-count": 191, "name": "Felipe Beach", "followers-count": 56 }, "sender-location": point("38.59,75.94"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "sprint", "customization" }}, "message-text": " like sprint the customization is awesome:)", "countA": 96, "countB": 66 }
+{ "tweetid": 97i64, "user": { "screen-name": "MaximaPoehl$770", "lang": "en", "friends-count": 9, "statuses-count": 99, "name": "Maxima Poehl", "followers-count": 198 }, "sender-location": point("46.94,66.2"), "send-time": datetime("2008-03-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " like iphone the customization is good", "countA": 97, "countB": 83 }
+{ "tweetid": 98i64, "user": { "screen-name": "IraLombardi#278", "lang": "en", "friends-count": 10, "statuses-count": 282, "name": "Ira Lombardi", "followers-count": 26 }, "sender-location": point("44.99,93.61"), "send-time": datetime("2011-02-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " love iphone its platform is awesome:)", "countA": 98, "countB": 78 }
+{ "tweetid": 99i64, "user": { "screen-name": "RexHincken_917", "lang": "en", "friends-count": 88, "statuses-count": 292, "name": "Rex Hincken", "followers-count": 74 }, "sender-location": point("42.0,81.22"), "send-time": datetime("2008-09-01T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voicemail-service" }}, "message-text": " love t-mobile the voicemail-service is amazing:)", "countA": 99, "countB": 51 }
+{ "tweetid": 100i64, "user": { "screen-name": "DakotaTeagarden_163", "lang": "en", "friends-count": 54, "statuses-count": 391, "name": "Dakota Teagarden", "followers-count": 160 }, "sender-location": point("43.59,92.49"), "send-time": datetime("2010-11-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "shortcut-menu" }}, "message-text": " can't stand samsung its shortcut-menu is OMG:(", "countA": 100, "countB": 184 }
+{ "tweetid": 101i64, "user": { "screen-name": "ChetMilliron_934", "lang": "en", "friends-count": 62, "statuses-count": 453, "name": "Chet Milliron", "followers-count": 53 }, "sender-location": point("47.95,77.58"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " like samsung the 3G is mind-blowing:)", "countA": 101, "countB": 142 }
+{ "tweetid": 102i64, "user": { "screen-name": "ZackLosey_956", "lang": "en", "friends-count": 90, "statuses-count": 6, "name": "Zack Losey", "followers-count": 116 }, "sender-location": point("26.53,80.6"), "send-time": datetime("2005-01-11T10:10:00.000Z"), "referred-topics": {{ "verizon", "platform" }}, "message-text": " like verizon its platform is amazing", "countA": 102, "countB": 190 }
+{ "tweetid": 103i64, "user": { "screen-name": "BrionyLafortune$483", "lang": "en", "friends-count": 87, "statuses-count": 496, "name": "Briony Lafortune", "followers-count": 4 }, "sender-location": point("42.2,73.96"), "send-time": datetime("2010-12-05T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " can't stand samsung its voice-command is OMG", "countA": 103, "countB": 27 }
+{ "tweetid": 104i64, "user": { "screen-name": "SaraGraham@726", "lang": "en", "friends-count": 38, "statuses-count": 398, "name": "Sara Graham", "followers-count": 68 }, "sender-location": point("32.83,81.29"), "send-time": datetime("2009-10-25T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " like iphone its platform is awesome", "countA": 104, "countB": 55 }
+{ "tweetid": 105i64, "user": { "screen-name": "EvanBarnes_217", "lang": "en", "friends-count": 42, "statuses-count": 239, "name": "Evan Barnes", "followers-count": 108 }, "sender-location": point("44.7,90.98"), "send-time": datetime("2008-07-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " like sprint its reachability is mind-blowing", "countA": 105, "countB": 3 }
+{ "tweetid": 106i64, "user": { "screen-name": "JulianeNorthey#34", "lang": "en", "friends-count": 69, "statuses-count": 94, "name": "Juliane Northey", "followers-count": 187 }, "sender-location": point("26.32,67.64"), "send-time": datetime("2007-06-26T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " can't stand sprint the voice-clarity is terrible", "countA": 106, "countB": 127 }
+{ "tweetid": 107i64, "user": { "screen-name": "ShannahBailey$196", "lang": "en", "friends-count": 47, "statuses-count": 215, "name": "Shannah Bailey", "followers-count": 67 }, "sender-location": point("40.85,87.01"), "send-time": datetime("2009-10-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " love verizon the reachability is awesome:)", "countA": 107, "countB": 31 }
+{ "tweetid": 108i64, "user": { "screen-name": "GranvilleKnisely$497", "lang": "en", "friends-count": 57, "statuses-count": 117, "name": "Granville Knisely", "followers-count": 52 }, "sender-location": point("35.46,78.27"), "send-time": datetime("2006-06-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " hate samsung its customization is horrible", "countA": 108, "countB": 148 }
+{ "tweetid": 109i64, "user": { "screen-name": "LeonardoJardine@763", "lang": "en", "friends-count": 48, "statuses-count": 415, "name": "Leonardo Jardine", "followers-count": 96 }, "sender-location": point("27.7,92.32"), "send-time": datetime("2010-12-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " dislike verizon the signal is bad", "countA": 109, "countB": 29 }
+{ "tweetid": 110i64, "user": { "screen-name": "AuroraMcelroy@927", "lang": "en", "friends-count": 79, "statuses-count": 297, "name": "Aurora Mcelroy", "followers-count": 119 }, "sender-location": point("48.56,85.12"), "send-time": datetime("2005-06-14T10:10:00.000Z"), "referred-topics": {{ "motorola", "shortcut-menu" }}, "message-text": " dislike motorola the shortcut-menu is bad:(", "countA": 110, "countB": 157 }
+{ "tweetid": 111i64, "user": { "screen-name": "NoleneLeslie#166", "lang": "en", "friends-count": 30, "statuses-count": 3, "name": "Nolene Leslie", "followers-count": 18 }, "sender-location": point("31.07,78.53"), "send-time": datetime("2005-10-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "platform" }}, "message-text": " like t-mobile its platform is good", "countA": 111, "countB": 50 }
+{ "tweetid": 112i64, "user": { "screen-name": "EusebioBeedell@329", "lang": "en", "friends-count": 94, "statuses-count": 341, "name": "Eusebio Beedell", "followers-count": 89 }, "sender-location": point("32.75,68.79"), "send-time": datetime("2007-08-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " can't stand verizon its network is terrible:(", "countA": 112, "countB": 30 }
+{ "tweetid": 113i64, "user": { "screen-name": "WoodySaltser$873", "lang": "en", "friends-count": 68, "statuses-count": 365, "name": "Woody Saltser", "followers-count": 132 }, "sender-location": point("37.57,88.05"), "send-time": datetime("2012-01-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voice-clarity" }}, "message-text": " love t-mobile its voice-clarity is awesome", "countA": 113, "countB": 177 }
+{ "tweetid": 114i64, "user": { "screen-name": "ReannaSeelig#553", "lang": "en", "friends-count": 31, "statuses-count": 291, "name": "Reanna Seelig", "followers-count": 175 }, "sender-location": point("42.87,72.38"), "send-time": datetime("2006-03-20T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " dislike motorola the signal is OMG:(", "countA": 114, "countB": 59 }
+{ "tweetid": 115i64, "user": { "screen-name": "AllannahNapier@336", "lang": "en", "friends-count": 34, "statuses-count": 359, "name": "Allannah Napier", "followers-count": 50 }, "sender-location": point("31.29,88.73"), "send-time": datetime("2010-07-28T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " hate samsung the wireless is bad:(", "countA": 115, "countB": 175 }
+{ "tweetid": 116i64, "user": { "screen-name": "AlaynaOsteen_327", "lang": "en", "friends-count": 59, "statuses-count": 237, "name": "Alayna Osteen", "followers-count": 12 }, "sender-location": point("30.6,71.97"), "send-time": datetime("2007-07-25T10:10:00.000Z"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is amazing:)", "countA": 116, "countB": 70 }
+{ "tweetid": 117i64, "user": { "screen-name": "LeticiaMillard#139", "lang": "en", "friends-count": 95, "statuses-count": 46, "name": "Leticia Millard", "followers-count": 72 }, "sender-location": point("26.53,73.37"), "send-time": datetime("2005-06-22T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " dislike iphone its 3G is horrible", "countA": 117, "countB": 168 }
+{ "tweetid": 118i64, "user": { "screen-name": "WinifredMckee_639", "lang": "en", "friends-count": 48, "statuses-count": 442, "name": "Winifred Mckee", "followers-count": 199 }, "sender-location": point("27.51,76.65"), "send-time": datetime("2012-06-19T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " can't stand sprint the reachability is bad", "countA": 118, "countB": 70 }
+{ "tweetid": 119i64, "user": { "screen-name": "SungShea#585", "lang": "en", "friends-count": 38, "statuses-count": 193, "name": "Sung Shea", "followers-count": 149 }, "sender-location": point("28.86,83.73"), "send-time": datetime("2009-04-22T10:10:00.000Z"), "referred-topics": {{ "samsung", "customer-service" }}, "message-text": " can't stand samsung its customer-service is horrible:(", "countA": 119, "countB": 155 }
+{ "tweetid": 120i64, "user": { "screen-name": "BernadineSutton@199", "lang": "en", "friends-count": 72, "statuses-count": 46, "name": "Bernadine Sutton", "followers-count": 105 }, "sender-location": point("40.19,77.94"), "send-time": datetime("2007-11-12T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " like sprint the voice-clarity is good:)", "countA": 120, "countB": 63 }
+{ "tweetid": 121i64, "user": { "screen-name": "DeedeeJerome#182", "lang": "en", "friends-count": 74, "statuses-count": 342, "name": "Deedee Jerome", "followers-count": 170 }, "sender-location": point("42.66,73.84"), "send-time": datetime("2012-07-03T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " can't stand sprint its reachability is horrible", "countA": 121, "countB": 163 }
+{ "tweetid": 122i64, "user": { "screen-name": "NigelPrechtl$759", "lang": "en", "friends-count": 10, "statuses-count": 133, "name": "Nigel Prechtl", "followers-count": 137 }, "sender-location": point("37.22,80.92"), "send-time": datetime("2012-01-05T10:10:00.000Z"), "referred-topics": {{ "at&t", "speed" }}, "message-text": " dislike at&t its speed is terrible", "countA": 122, "countB": 23 }
+{ "tweetid": 123i64, "user": { "screen-name": "KimmyWynne$198", "lang": "en", "friends-count": 16, "statuses-count": 40, "name": "Kimmy Wynne", "followers-count": 6 }, "sender-location": point("26.49,70.55"), "send-time": datetime("2008-10-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " dislike t-mobile the signal is terrible", "countA": 123, "countB": 44 }
+{ "tweetid": 124i64, "user": { "screen-name": "ByronHarshman$352", "lang": "en", "friends-count": 26, "statuses-count": 133, "name": "Byron Harshman", "followers-count": 144 }, "sender-location": point("26.4,88.43"), "send-time": datetime("2012-03-21T10:10:00.000Z"), "referred-topics": {{ "samsung", "touch-screen" }}, "message-text": " dislike samsung the touch-screen is terrible", "countA": 124, "countB": 44 }
+{ "tweetid": 125i64, "user": { "screen-name": "PlacidPrevatt#865", "lang": "en", "friends-count": 3, "statuses-count": 493, "name": "Placid Prevatt", "followers-count": 10 }, "sender-location": point("43.09,84.0"), "send-time": datetime("2010-07-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "voicemail-service" }}, "message-text": " love motorola its voicemail-service is amazing:)", "countA": 125, "countB": 7 }
+{ "tweetid": 126i64, "user": { "screen-name": "TranterGarneis_456", "lang": "en", "friends-count": 89, "statuses-count": 151, "name": "Tranter Garneis", "followers-count": 166 }, "sender-location": point("41.6,93.6"), "send-time": datetime("2007-08-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone its shortcut-menu is amazing:)", "countA": 126, "countB": 76 }
+{ "tweetid": 127i64, "user": { "screen-name": "DonyaWilliamson$23", "lang": "en", "friends-count": 62, "statuses-count": 325, "name": "Donya Williamson", "followers-count": 101 }, "sender-location": point("27.75,66.01"), "send-time": datetime("2005-11-02T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " can't stand verizon its reachability is OMG", "countA": 127, "countB": 184 }
+{ "tweetid": 128i64, "user": { "screen-name": "GalinaJoghs$90", "lang": "en", "friends-count": 61, "statuses-count": 86, "name": "Galina Joghs", "followers-count": 169 }, "sender-location": point("30.95,71.04"), "send-time": datetime("2010-06-01T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " can't stand motorola its signal is horrible", "countA": 128, "countB": 24 }
+{ "tweetid": 129i64, "user": { "screen-name": "SamsonWerner#683", "lang": "en", "friends-count": 92, "statuses-count": 171, "name": "Samson Werner", "followers-count": 108 }, "sender-location": point("36.53,92.04"), "send-time": datetime("2009-08-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " hate motorola its network is terrible:(", "countA": 129, "countB": 80 }
+{ "tweetid": 130i64, "user": { "screen-name": "GabrielleMang#424", "lang": "en", "friends-count": 66, "statuses-count": 8, "name": "Gabrielle Mang", "followers-count": 80 }, "sender-location": point("36.74,96.64"), "send-time": datetime("2006-04-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " love sprint its plan is amazing:)", "countA": 130, "countB": 157 }
+{ "tweetid": 131i64, "user": { "screen-name": "ZachariasBaldwin#74", "lang": "en", "friends-count": 5, "statuses-count": 205, "name": "Zacharias Baldwin", "followers-count": 87 }, "sender-location": point("25.44,72.7"), "send-time": datetime("2006-01-21T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " like sprint the platform is amazing", "countA": 131, "countB": 192 }
+{ "tweetid": 132i64, "user": { "screen-name": "FanniePoorbaugh@315", "lang": "en", "friends-count": 59, "statuses-count": 441, "name": "Fannie Poorbaugh", "followers-count": 114 }, "sender-location": point("48.89,68.65"), "send-time": datetime("2005-05-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " hate verizon the reachability is terrible", "countA": 132, "countB": 142 }
+{ "tweetid": 133i64, "user": { "screen-name": "SandraTeagarden$747", "lang": "en", "friends-count": 60, "statuses-count": 353, "name": "Sandra Teagarden", "followers-count": 141 }, "sender-location": point("44.37,76.54"), "send-time": datetime("2008-05-25T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t its platform is bad", "countA": 133, "countB": 17 }
+{ "tweetid": 134i64, "user": { "screen-name": "SteveMayers_702", "lang": "en", "friends-count": 70, "statuses-count": 196, "name": "Steve Mayers", "followers-count": 22 }, "sender-location": point("41.52,91.39"), "send-time": datetime("2012-08-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "speed" }}, "message-text": " love iphone its speed is amazing", "countA": 134, "countB": 86 }
+{ "tweetid": 135i64, "user": { "screen-name": "RosalynPullman@789", "lang": "en", "friends-count": 17, "statuses-count": 470, "name": "Rosalyn Pullman", "followers-count": 123 }, "sender-location": point("48.33,86.41"), "send-time": datetime("2009-12-08T10:10:00.000Z"), "referred-topics": {{ "motorola", "customization" }}, "message-text": " hate motorola its customization is bad:(", "countA": 135, "countB": 171 }
+{ "tweetid": 136i64, "user": { "screen-name": "LamarChauvin$832", "lang": "en", "friends-count": 21, "statuses-count": 234, "name": "Lamar Chauvin", "followers-count": 184 }, "sender-location": point("36.83,89.48"), "send-time": datetime("2011-06-26T10:10:00.000Z"), "referred-topics": {{ "samsung", "reachability" }}, "message-text": " love samsung its reachability is awesome:)", "countA": 136, "countB": 77 }
+{ "tweetid": 137i64, "user": { "screen-name": "EleaseReade#477", "lang": "en", "friends-count": 24, "statuses-count": 299, "name": "Elease Reade", "followers-count": 24 }, "sender-location": point("45.55,93.09"), "send-time": datetime("2012-07-19T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " hate at&t the network is bad:(", "countA": 137, "countB": 25 }
+{ "tweetid": 138i64, "user": { "screen-name": "LeviPhilbrick$328", "lang": "en", "friends-count": 73, "statuses-count": 77, "name": "Levi Philbrick", "followers-count": 179 }, "sender-location": point("33.51,68.88"), "send-time": datetime("2009-08-28T10:10:00.000Z"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " dislike verizon its shortcut-menu is horrible", "countA": 138, "countB": 5 }
+{ "tweetid": 139i64, "user": { "screen-name": "RaeburnNickolson_295", "lang": "en", "friends-count": 72, "statuses-count": 176, "name": "Raeburn Nickolson", "followers-count": 103 }, "sender-location": point("38.42,74.16"), "send-time": datetime("2008-05-28T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " like at&t its voicemail-service is good:)", "countA": 139, "countB": 175 }
+{ "tweetid": 140i64, "user": { "screen-name": "NerissaBallou@177", "lang": "en", "friends-count": 17, "statuses-count": 447, "name": "Nerissa Ballou", "followers-count": 183 }, "sender-location": point("34.65,81.44"), "send-time": datetime("2008-07-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " hate at&t its plan is OMG", "countA": 140, "countB": 192 }
+{ "tweetid": 141i64, "user": { "screen-name": "DanyelWalker#602", "lang": "en", "friends-count": 22, "statuses-count": 397, "name": "Danyel Walker", "followers-count": 154 }, "sender-location": point("40.38,78.39"), "send-time": datetime("2012-01-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "3G" }}, "message-text": " hate t-mobile its 3G is OMG", "countA": 141, "countB": 9 }
+{ "tweetid": 142i64, "user": { "screen-name": "YaronLeichter_45", "lang": "en", "friends-count": 80, "statuses-count": 103, "name": "Yaron Leichter", "followers-count": 115 }, "sender-location": point("42.59,71.72"), "send-time": datetime("2010-04-24T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " like verizon the signal is mind-blowing:)", "countA": 142, "countB": 130 }
+{ "tweetid": 143i64, "user": { "screen-name": "DeemerCable_599", "lang": "en", "friends-count": 26, "statuses-count": 371, "name": "Deemer Cable", "followers-count": 34 }, "sender-location": point("29.89,70.29"), "send-time": datetime("2012-08-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " like motorola the wireless is awesome", "countA": 143, "countB": 50 }
+{ "tweetid": 144i64, "user": { "screen-name": "AliaHay_860", "lang": "en", "friends-count": 34, "statuses-count": 470, "name": "Alia Hay", "followers-count": 111 }, "sender-location": point("46.39,96.22"), "send-time": datetime("2010-12-18T10:10:00.000Z"), "referred-topics": {{ "at&t", "shortcut-menu" }}, "message-text": " like at&t its shortcut-menu is amazing:)", "countA": 144, "countB": 184 }
+{ "tweetid": 145i64, "user": { "screen-name": "AdamMoore$384", "lang": "en", "friends-count": 42, "statuses-count": 198, "name": "Adam Moore", "followers-count": 17 }, "sender-location": point("27.99,84.55"), "send-time": datetime("2007-03-16T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " love sprint its network is good", "countA": 145, "countB": 162 }
+{ "tweetid": 146i64, "user": { "screen-name": "AileenSouthern$868", "lang": "en", "friends-count": 62, "statuses-count": 388, "name": "Aileen Southern", "followers-count": 131 }, "sender-location": point("30.85,66.62"), "send-time": datetime("2005-07-14T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " like t-mobile the signal is amazing", "countA": 146, "countB": 102 }
+{ "tweetid": 147i64, "user": { "screen-name": "JewellWise_154", "lang": "en", "friends-count": 2, "statuses-count": 279, "name": "Jewell Wise", "followers-count": 107 }, "sender-location": point("46.72,83.98"), "send-time": datetime("2006-01-09T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " hate motorola the plan is terrible:(", "countA": 147, "countB": 174 }
+{ "tweetid": 148i64, "user": { "screen-name": "DanielJowers#519", "lang": "en", "friends-count": 23, "statuses-count": 22, "name": "Daniel Jowers", "followers-count": 131 }, "sender-location": point("34.26,72.22"), "send-time": datetime("2008-07-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-clarity" }}, "message-text": " like motorola its voice-clarity is mind-blowing:)", "countA": 148, "countB": 6 }
+{ "tweetid": 149i64, "user": { "screen-name": "DillonWilliams_557", "lang": "en", "friends-count": 18, "statuses-count": 136, "name": "Dillon Williams", "followers-count": 35 }, "sender-location": point("46.63,97.38"), "send-time": datetime("2011-05-09T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " love motorola the network is good", "countA": 149, "countB": 20 }
+{ "tweetid": 150i64, "user": { "screen-name": "DerrickBullard$202", "lang": "en", "friends-count": 16, "statuses-count": 9, "name": "Derrick Bullard", "followers-count": 100 }, "sender-location": point("41.89,90.62"), "send-time": datetime("2012-08-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " like t-mobile its touch-screen is mind-blowing", "countA": 150, "countB": 145 }
+{ "tweetid": 151i64, "user": { "screen-name": "LuigiMcfall_976", "lang": "en", "friends-count": 31, "statuses-count": 215, "name": "Luigi Mcfall", "followers-count": 79 }, "sender-location": point("45.38,70.52"), "send-time": datetime("2005-11-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " hate motorola its reachability is OMG:(", "countA": 151, "countB": 43 }
+{ "tweetid": 152i64, "user": { "screen-name": "MartinPinney_858", "lang": "en", "friends-count": 21, "statuses-count": 465, "name": "Martin Pinney", "followers-count": 16 }, "sender-location": point("32.87,75.66"), "send-time": datetime("2007-10-12T10:10:00.000Z"), "referred-topics": {{ "at&t", "reachability" }}, "message-text": " hate at&t its reachability is terrible", "countA": 152, "countB": 56 }
+{ "tweetid": 153i64, "user": { "screen-name": "JackieAft_623", "lang": "en", "friends-count": 70, "statuses-count": 413, "name": "Jackie Aft", "followers-count": 138 }, "sender-location": point("29.9,73.29"), "send-time": datetime("2010-07-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " like verizon the voice-clarity is amazing", "countA": 153, "countB": 154 }
+{ "tweetid": 154i64, "user": { "screen-name": "SherriWickes#118", "lang": "en", "friends-count": 20, "statuses-count": 31, "name": "Sherri Wickes", "followers-count": 59 }, "sender-location": point("39.2,79.2"), "send-time": datetime("2006-07-13T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " hate sprint the network is OMG:(", "countA": 154, "countB": 124 }
+{ "tweetid": 155i64, "user": { "screen-name": "CarlieCowher@103", "lang": "en", "friends-count": 81, "statuses-count": 127, "name": "Carlie Cowher", "followers-count": 184 }, "sender-location": point("30.3,76.43"), "send-time": datetime("2010-10-04T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " can't stand sprint the voice-clarity is bad", "countA": 155, "countB": 39 }
+{ "tweetid": 156i64, "user": { "screen-name": "AndraWardle@74", "lang": "en", "friends-count": 41, "statuses-count": 35, "name": "Andra Wardle", "followers-count": 168 }, "sender-location": point("45.49,93.97"), "send-time": datetime("2009-02-18T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " love t-mobile its wireless is amazing:)", "countA": 156, "countB": 23 }
+{ "tweetid": 157i64, "user": { "screen-name": "KanishaPinney@150", "lang": "en", "friends-count": 89, "statuses-count": 315, "name": "Kanisha Pinney", "followers-count": 173 }, "sender-location": point("24.72,77.36"), "send-time": datetime("2005-06-10T10:10:00.000Z"), "referred-topics": {{ "verizon", "touch-screen" }}, "message-text": " like verizon the touch-screen is amazing", "countA": 157, "countB": 153 }
+{ "tweetid": 158i64, "user": { "screen-name": "GlyndaSchere@104", "lang": "en", "friends-count": 6, "statuses-count": 111, "name": "Glynda Schere", "followers-count": 120 }, "sender-location": point("33.86,67.49"), "send-time": datetime("2010-11-26T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " like verizon its reachability is amazing:)", "countA": 158, "countB": 96 }
+{ "tweetid": 159i64, "user": { "screen-name": "JenelleNehling@461", "lang": "en", "friends-count": 4, "statuses-count": 384, "name": "Jenelle Nehling", "followers-count": 57 }, "sender-location": point("32.65,89.38"), "send-time": datetime("2010-12-06T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone its shortcut-menu is good", "countA": 159, "countB": 113 }
+{ "tweetid": 160i64, "user": { "screen-name": "DelWheeler@286", "lang": "en", "friends-count": 23, "statuses-count": 403, "name": "Del Wheeler", "followers-count": 177 }, "sender-location": point("39.39,78.05"), "send-time": datetime("2011-07-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " love samsung the platform is mind-blowing", "countA": 160, "countB": 163 }
+{ "tweetid": 161i64, "user": { "screen-name": "TrinityCowart@360", "lang": "en", "friends-count": 85, "statuses-count": 204, "name": "Trinity Cowart", "followers-count": 145 }, "sender-location": point("32.11,76.64"), "send-time": datetime("2010-10-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " like t-mobile its wireless is amazing:)", "countA": 161, "countB": 62 }
+{ "tweetid": 162i64, "user": { "screen-name": "HudsonBasmanoff_348", "lang": "en", "friends-count": 3, "statuses-count": 394, "name": "Hudson Basmanoff", "followers-count": 114 }, "sender-location": point("39.24,78.13"), "send-time": datetime("2007-09-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " can't stand motorola its wireless is OMG:(", "countA": 162, "countB": 57 }
+{ "tweetid": 163i64, "user": { "screen-name": "MatthewPowers_801", "lang": "en", "friends-count": 2, "statuses-count": 203, "name": "Matthew Powers", "followers-count": 199 }, "sender-location": point("33.79,69.57"), "send-time": datetime("2012-04-05T10:10:00.000Z"), "referred-topics": {{ "at&t", "customer-service" }}, "message-text": " love at&t its customer-service is awesome", "countA": 163, "countB": 115 }
+{ "tweetid": 164i64, "user": { "screen-name": "TitaniaKern$100", "lang": "en", "friends-count": 98, "statuses-count": 300, "name": "Titania Kern", "followers-count": 118 }, "sender-location": point("45.86,67.64"), "send-time": datetime("2005-11-04T10:10:00.000Z"), "referred-topics": {{ "at&t", "shortcut-menu" }}, "message-text": " love at&t the shortcut-menu is amazing", "countA": 164, "countB": 148 }
+{ "tweetid": 165i64, "user": { "screen-name": "EhtelCrissman#778", "lang": "en", "friends-count": 33, "statuses-count": 286, "name": "Ehtel Crissman", "followers-count": 63 }, "sender-location": point("32.98,82.49"), "send-time": datetime("2011-08-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is good:)", "countA": 165, "countB": 127 }
+{ "tweetid": 166i64, "user": { "screen-name": "WilletteLeslie@682", "lang": "en", "friends-count": 38, "statuses-count": 491, "name": "Willette Leslie", "followers-count": 75 }, "sender-location": point("32.57,84.97"), "send-time": datetime("2012-04-18T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " like verizon its reachability is mind-blowing", "countA": 166, "countB": 162 }
+{ "tweetid": 167i64, "user": { "screen-name": "DarellHincken_722", "lang": "en", "friends-count": 33, "statuses-count": 111, "name": "Darell Hincken", "followers-count": 74 }, "sender-location": point("48.57,77.77"), "send-time": datetime("2008-06-27T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile its speed is good:)", "countA": 167, "countB": 93 }
+{ "tweetid": 168i64, "user": { "screen-name": "DuaneKing@956", "lang": "en", "friends-count": 5, "statuses-count": 44, "name": "Duane King", "followers-count": 169 }, "sender-location": point("31.26,68.61"), "send-time": datetime("2011-03-24T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " like sprint its touch-screen is mind-blowing:)", "countA": 168, "countB": 174 }
+{ "tweetid": 169i64, "user": { "screen-name": "AmbroseKeilbach$300", "lang": "en", "friends-count": 76, "statuses-count": 278, "name": "Ambrose Keilbach", "followers-count": 54 }, "sender-location": point("29.75,71.35"), "send-time": datetime("2012-02-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "signal" }}, "message-text": " can't stand sprint its signal is horrible", "countA": 169, "countB": 129 }
+{ "tweetid": 170i64, "user": { "screen-name": "KarlBrooks#97", "lang": "en", "friends-count": 76, "statuses-count": 150, "name": "Karl Brooks", "followers-count": 117 }, "sender-location": point("30.77,85.78"), "send-time": datetime("2006-09-16T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " hate motorola the plan is horrible", "countA": 170, "countB": 174 }
+{ "tweetid": 171i64, "user": { "screen-name": "ShainaMayers$261", "lang": "en", "friends-count": 76, "statuses-count": 240, "name": "Shaina Mayers", "followers-count": 194 }, "sender-location": point("26.11,78.33"), "send-time": datetime("2005-06-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "voicemail-service" }}, "message-text": " love sprint its voicemail-service is mind-blowing:)", "countA": 171, "countB": 69 }
+{ "tweetid": 172i64, "user": { "screen-name": "LakeshaPery_35", "lang": "en", "friends-count": 58, "statuses-count": 300, "name": "Lakesha Pery", "followers-count": 51 }, "sender-location": point("38.45,75.31"), "send-time": datetime("2009-12-20T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " like motorola its 3G is good:)", "countA": 172, "countB": 127 }
+{ "tweetid": 173i64, "user": { "screen-name": "DoranMingle#901", "lang": "en", "friends-count": 3, "statuses-count": 302, "name": "Doran Mingle", "followers-count": 152 }, "sender-location": point("47.76,91.28"), "send-time": datetime("2009-06-07T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " dislike at&t the customization is OMG", "countA": 173, "countB": 41 }
+{ "tweetid": 174i64, "user": { "screen-name": "AmadaHatcher#710", "lang": "en", "friends-count": 3, "statuses-count": 12, "name": "Amada Hatcher", "followers-count": 193 }, "sender-location": point("28.64,89.42"), "send-time": datetime("2011-09-16T10:10:00.000Z"), "referred-topics": {{ "samsung", "network" }}, "message-text": " hate samsung the network is OMG", "countA": 174, "countB": 0 }
+{ "tweetid": 175i64, "user": { "screen-name": "CorianderMoon@658", "lang": "en", "friends-count": 28, "statuses-count": 117, "name": "Coriander Moon", "followers-count": 77 }, "sender-location": point("43.82,87.23"), "send-time": datetime("2011-03-26T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " dislike samsung the customization is bad:(", "countA": 175, "countB": 5 }
+{ "tweetid": 176i64, "user": { "screen-name": "PhilomenaEiford#608", "lang": "en", "friends-count": 53, "statuses-count": 467, "name": "Philomena Eiford", "followers-count": 80 }, "sender-location": point("30.06,71.08"), "send-time": datetime("2006-05-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " like motorola the signal is amazing", "countA": 176, "countB": 89 }
+{ "tweetid": 177i64, "user": { "screen-name": "JacobCongdon$162", "lang": "en", "friends-count": 61, "statuses-count": 161, "name": "Jacob Congdon", "followers-count": 19 }, "sender-location": point("36.36,78.43"), "send-time": datetime("2006-12-10T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " dislike samsung its platform is horrible", "countA": 177, "countB": 15 }
+{ "tweetid": 178i64, "user": { "screen-name": "GeorgeRichards$777", "lang": "en", "friends-count": 12, "statuses-count": 213, "name": "George Richards", "followers-count": 72 }, "sender-location": point("44.78,90.69"), "send-time": datetime("2006-03-17T10:10:00.000Z"), "referred-topics": {{ "verizon", "speed" }}, "message-text": " can't stand verizon its speed is horrible", "countA": 178, "countB": 64 }
+{ "tweetid": 179i64, "user": { "screen-name": "BridgerHamilton@431", "lang": "en", "friends-count": 51, "statuses-count": 396, "name": "Bridger Hamilton", "followers-count": 110 }, "sender-location": point("32.82,81.54"), "send-time": datetime("2008-12-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " love verizon the plan is awesome", "countA": 179, "countB": 141 }
+{ "tweetid": 180i64, "user": { "screen-name": "FemieLucy@34", "lang": "en", "friends-count": 68, "statuses-count": 221, "name": "Femie Lucy", "followers-count": 11 }, "sender-location": point("31.58,82.78"), "send-time": datetime("2010-02-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " hate iphone its customization is OMG", "countA": 180, "countB": 191 }
+{ "tweetid": 181i64, "user": { "screen-name": "JodiNapier@338", "lang": "en", "friends-count": 96, "statuses-count": 467, "name": "Jodi Napier", "followers-count": 69 }, "sender-location": point("26.14,78.5"), "send-time": datetime("2007-01-06T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " love verizon the plan is mind-blowing", "countA": 181, "countB": 13 }
+{ "tweetid": 182i64, "user": { "screen-name": "MitsueRawls_424", "lang": "en", "friends-count": 91, "statuses-count": 70, "name": "Mitsue Rawls", "followers-count": 193 }, "sender-location": point("47.77,70.41"), "send-time": datetime("2007-04-12T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " dislike iphone its customer-service is OMG", "countA": 182, "countB": 59 }
+{ "tweetid": 183i64, "user": { "screen-name": "DeshawnAultman_690", "lang": "en", "friends-count": 49, "statuses-count": 330, "name": "Deshawn Aultman", "followers-count": 39 }, "sender-location": point("40.65,79.37"), "send-time": datetime("2005-08-11T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " like sprint the shortcut-menu is amazing:)", "countA": 183, "countB": 199 }
+{ "tweetid": 184i64, "user": { "screen-name": "BradfordEiford#127", "lang": "en", "friends-count": 90, "statuses-count": 425, "name": "Bradford Eiford", "followers-count": 10 }, "sender-location": point("40.85,91.17"), "send-time": datetime("2005-09-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "voicemail-service" }}, "message-text": " dislike iphone its voicemail-service is horrible:(", "countA": 184, "countB": 141 }
+{ "tweetid": 185i64, "user": { "screen-name": "MadelynGaskins_356", "lang": "en", "friends-count": 48, "statuses-count": 455, "name": "Madelyn Gaskins", "followers-count": 66 }, "sender-location": point("33.81,88.32"), "send-time": datetime("2011-10-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " love at&t its voicemail-service is good:)", "countA": 185, "countB": 189 }
+{ "tweetid": 186i64, "user": { "screen-name": "SophiaMang@768", "lang": "en", "friends-count": 28, "statuses-count": 86, "name": "Sophia Mang", "followers-count": 125 }, "sender-location": point("37.45,68.47"), "send-time": datetime("2010-08-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "touch-screen" }}, "message-text": " love iphone its touch-screen is awesome:)", "countA": 186, "countB": 149 }
+{ "tweetid": 187i64, "user": { "screen-name": "VernonKnisely#170", "lang": "en", "friends-count": 63, "statuses-count": 406, "name": "Vernon Knisely", "followers-count": 31 }, "sender-location": point("32.93,94.65"), "send-time": datetime("2006-03-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " love sprint the reachability is awesome", "countA": 187, "countB": 98 }
+{ "tweetid": 188i64, "user": { "screen-name": "AmyEndsley@85", "lang": "en", "friends-count": 83, "statuses-count": 43, "name": "Amy Endsley", "followers-count": 7 }, "sender-location": point("27.33,82.34"), "send-time": datetime("2005-01-11T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " love at&t its customization is good", "countA": 188, "countB": 48 }
+{ "tweetid": 189i64, "user": { "screen-name": "AntonChristner#166", "lang": "en", "friends-count": 12, "statuses-count": 10, "name": "Anton Christner", "followers-count": 66 }, "sender-location": point("25.29,89.55"), "send-time": datetime("2006-11-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " can't stand motorola its wireless is terrible:(", "countA": 189, "countB": 160 }
+{ "tweetid": 190i64, "user": { "screen-name": "DeshawnHarris#34", "lang": "en", "friends-count": 32, "statuses-count": 488, "name": "Deshawn Harris", "followers-count": 178 }, "sender-location": point("45.46,76.04"), "send-time": datetime("2007-05-13T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " like samsung the customization is awesome:)", "countA": 190, "countB": 86 }
+{ "tweetid": 191i64, "user": { "screen-name": "MarioHolts_870", "lang": "en", "friends-count": 20, "statuses-count": 192, "name": "Mario Holts", "followers-count": 71 }, "sender-location": point("29.69,71.42"), "send-time": datetime("2005-09-15T10:10:00.000Z"), "referred-topics": {{ "iphone", "speed" }}, "message-text": " hate iphone the speed is horrible:(", "countA": 191, "countB": 150 }
+{ "tweetid": 192i64, "user": { "screen-name": "DeanHall#220", "lang": "en", "friends-count": 55, "statuses-count": 236, "name": "Dean Hall", "followers-count": 68 }, "sender-location": point("48.12,72.0"), "send-time": datetime("2006-11-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " love sprint the platform is awesome", "countA": 192, "countB": 199 }
+{ "tweetid": 193i64, "user": { "screen-name": "LoanClarke_206", "lang": "en", "friends-count": 32, "statuses-count": 173, "name": "Loan Clarke", "followers-count": 186 }, "sender-location": point("45.39,96.01"), "send-time": datetime("2009-05-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "customization" }}, "message-text": " can't stand verizon its customization is horrible:(", "countA": 193, "countB": 147 }
+{ "tweetid": 194i64, "user": { "screen-name": "DonnetteGoodman@627", "lang": "en", "friends-count": 61, "statuses-count": 202, "name": "Donnette Goodman", "followers-count": 106 }, "sender-location": point("44.72,73.13"), "send-time": datetime("2005-04-27T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "plan" }}, "message-text": " love t-mobile its plan is good", "countA": 194, "countB": 142 }
+{ "tweetid": 195i64, "user": { "screen-name": "TamekaPorter#315", "lang": "en", "friends-count": 79, "statuses-count": 63, "name": "Tameka Porter", "followers-count": 7 }, "sender-location": point("37.68,81.78"), "send-time": datetime("2006-06-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " like samsung its wireless is good", "countA": 195, "countB": 126 }
+{ "tweetid": 196i64, "user": { "screen-name": "GarlandClark@425", "lang": "en", "friends-count": 15, "statuses-count": 375, "name": "Garland Clark", "followers-count": 24 }, "sender-location": point("44.9,70.1"), "send-time": datetime("2006-01-08T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint the touch-screen is OMG:(", "countA": 196, "countB": 197 }
+{ "tweetid": 197i64, "user": { "screen-name": "RupertSanner$868", "lang": "en", "friends-count": 49, "statuses-count": 414, "name": "Rupert Sanner", "followers-count": 189 }, "sender-location": point("40.45,94.94"), "send-time": datetime("2012-06-20T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " can't stand iphone the customer-service is terrible:(", "countA": 197, "countB": 195 }
+{ "tweetid": 198i64, "user": { "screen-name": "JenZoucks#841", "lang": "en", "friends-count": 99, "statuses-count": 73, "name": "Jen Zoucks", "followers-count": 71 }, "sender-location": point("38.77,70.33"), "send-time": datetime("2010-07-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "signal" }}, "message-text": " like iphone its signal is good", "countA": 198, "countB": 94 }
+{ "tweetid": 199i64, "user": { "screen-name": "IsmaelLlora@529", "lang": "en", "friends-count": 63, "statuses-count": 239, "name": "Ismael Llora", "followers-count": 135 }, "sender-location": point("35.31,82.71"), "send-time": datetime("2007-03-19T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is amazing:)", "countA": 199, "countB": 172 }
+{ "tweetid": 200i64, "user": { "screen-name": "JenniferHoltzer_459", "lang": "en", "friends-count": 93, "statuses-count": 172, "name": "Jennifer Holtzer", "followers-count": 51 }, "sender-location": point("30.44,81.57"), "send-time": datetime("2007-11-24T10:10:00.000Z"), "referred-topics": {{ "at&t", "wireless" }}, "message-text": " love at&t the wireless is amazing", "countA": 200, "countB": 20 }
+{ "tweetid": 201i64, "user": { "screen-name": "EvanJardine$78", "lang": "en", "friends-count": 80, "statuses-count": 349, "name": "Evan Jardine", "followers-count": 155 }, "sender-location": point("39.12,68.0"), "send-time": datetime("2010-01-13T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " like t-mobile its signal is awesome:)", "countA": 201, "countB": 65 }
+{ "tweetid": 202i64, "user": { "screen-name": "KerenBard_268", "lang": "en", "friends-count": 28, "statuses-count": 348, "name": "Keren Bard", "followers-count": 158 }, "sender-location": point("32.72,77.86"), "send-time": datetime("2008-07-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " love sprint its platform is awesome:)", "countA": 202, "countB": 117 }
+{ "tweetid": 203i64, "user": { "screen-name": "GilbertLosey#710", "lang": "en", "friends-count": 67, "statuses-count": 141, "name": "Gilbert Losey", "followers-count": 192 }, "sender-location": point("44.66,84.32"), "send-time": datetime("2011-09-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "signal" }}, "message-text": " dislike at&t its signal is horrible", "countA": 203, "countB": 182 }
+{ "tweetid": 204i64, "user": { "screen-name": "TawnyGraham_816", "lang": "en", "friends-count": 63, "statuses-count": 13, "name": "Tawny Graham", "followers-count": 155 }, "sender-location": point("43.08,86.89"), "send-time": datetime("2012-05-10T10:10:00.000Z"), "referred-topics": {{ "verizon", "touch-screen" }}, "message-text": " dislike verizon its touch-screen is terrible", "countA": 204, "countB": 104 }
+{ "tweetid": 205i64, "user": { "screen-name": "LamontWilkins_977", "lang": "en", "friends-count": 19, "statuses-count": 102, "name": "Lamont Wilkins", "followers-count": 127 }, "sender-location": point("31.77,87.68"), "send-time": datetime("2005-04-12T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " dislike samsung its 3G is bad", "countA": 205, "countB": 57 }
+{ "tweetid": 206i64, "user": { "screen-name": "VincentRiggle$122", "lang": "en", "friends-count": 99, "statuses-count": 105, "name": "Vincent Riggle", "followers-count": 173 }, "sender-location": point("34.28,74.71"), "send-time": datetime("2007-05-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " like motorola the wireless is amazing:)", "countA": 206, "countB": 3 }
+{ "tweetid": 207i64, "user": { "screen-name": "AdelaJones_400", "lang": "en", "friends-count": 74, "statuses-count": 121, "name": "Adela Jones", "followers-count": 161 }, "sender-location": point("28.5,94.52"), "send-time": datetime("2010-01-20T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "platform" }}, "message-text": " like t-mobile the platform is good:)", "countA": 207, "countB": 110 }
+{ "tweetid": 208i64, "user": { "screen-name": "PrestonLittle$360", "lang": "en", "friends-count": 61, "statuses-count": 430, "name": "Preston Little", "followers-count": 115 }, "sender-location": point("35.79,78.52"), "send-time": datetime("2010-03-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " dislike sprint the speed is terrible:(", "countA": 208, "countB": 149 }
+{ "tweetid": 209i64, "user": { "screen-name": "TaraAnderson#214", "lang": "en", "friends-count": 60, "statuses-count": 219, "name": "Tara Anderson", "followers-count": 110 }, "sender-location": point("34.04,79.5"), "send-time": datetime("2006-12-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " dislike motorola its plan is OMG", "countA": 209, "countB": 164 }
+{ "tweetid": 210i64, "user": { "screen-name": "DeshawnWallace$305", "lang": "en", "friends-count": 29, "statuses-count": 279, "name": "Deshawn Wallace", "followers-count": 27 }, "sender-location": point("39.49,76.58"), "send-time": datetime("2011-01-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " dislike t-mobile the customization is OMG", "countA": 210, "countB": 70 }
+{ "tweetid": 211i64, "user": { "screen-name": "OliFisher$694", "lang": "en", "friends-count": 39, "statuses-count": 131, "name": "Oli Fisher", "followers-count": 101 }, "sender-location": point("44.91,94.72"), "send-time": datetime("2010-04-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " love iphone the 3G is mind-blowing", "countA": 211, "countB": 95 }
+{ "tweetid": 212i64, "user": { "screen-name": "LizaMathews#376", "lang": "en", "friends-count": 40, "statuses-count": 107, "name": "Liza Mathews", "followers-count": 70 }, "sender-location": point("46.01,77.85"), "send-time": datetime("2009-07-20T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " like verizon the plan is mind-blowing:)", "countA": 212, "countB": 189 }
+{ "tweetid": 213i64, "user": { "screen-name": "MylesRahl#433", "lang": "en", "friends-count": 51, "statuses-count": 144, "name": "Myles Rahl", "followers-count": 90 }, "sender-location": point("45.41,95.69"), "send-time": datetime("2006-04-03T10:10:00.000Z"), "referred-topics": {{ "verizon", "3G" }}, "message-text": " love verizon its 3G is mind-blowing:)", "countA": 213, "countB": 190 }
+{ "tweetid": 214i64, "user": { "screen-name": "BertEve@968", "lang": "en", "friends-count": 84, "statuses-count": 110, "name": "Bert Eve", "followers-count": 122 }, "sender-location": point("43.25,87.82"), "send-time": datetime("2009-11-07T10:10:00.000Z"), "referred-topics": {{ "at&t", "voice-command" }}, "message-text": " dislike at&t the voice-command is terrible:(", "countA": 214, "countB": 142 }
+{ "tweetid": 215i64, "user": { "screen-name": "MelissaLaurenzi_383", "lang": "en", "friends-count": 78, "statuses-count": 318, "name": "Melissa Laurenzi", "followers-count": 19 }, "sender-location": point("37.82,86.96"), "send-time": datetime("2010-05-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " like iphone its customer-service is mind-blowing", "countA": 215, "countB": 67 }
+{ "tweetid": 216i64, "user": { "screen-name": "ReneaPennington#175", "lang": "en", "friends-count": 42, "statuses-count": 16, "name": "Renea Pennington", "followers-count": 52 }, "sender-location": point("39.06,78.62"), "send-time": datetime("2006-05-23T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " dislike iphone its 3G is bad:(", "countA": 216, "countB": 50 }
+{ "tweetid": 217i64, "user": { "screen-name": "UrsulaMitchell@26", "lang": "en", "friends-count": 50, "statuses-count": 150, "name": "Ursula Mitchell", "followers-count": 191 }, "sender-location": point("34.02,71.3"), "send-time": datetime("2005-03-26T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " like iphone its customer-service is awesome", "countA": 217, "countB": 154 }
+{ "tweetid": 218i64, "user": { "screen-name": "WillyLambert_669", "lang": "en", "friends-count": 6, "statuses-count": 44, "name": "Willy Lambert", "followers-count": 60 }, "sender-location": point("32.84,73.25"), "send-time": datetime("2011-10-04T10:10:00.000Z"), "referred-topics": {{ "verizon", "customization" }}, "message-text": " like verizon the customization is amazing", "countA": 218, "countB": 135 }
+{ "tweetid": 219i64, "user": { "screen-name": "ChristineLaurence_912", "lang": "en", "friends-count": 77, "statuses-count": 458, "name": "Christine Laurence", "followers-count": 166 }, "sender-location": point("35.39,90.54"), "send-time": datetime("2009-12-01T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " like at&t the customization is awesome:)", "countA": 219, "countB": 170 }
+{ "tweetid": 220i64, "user": { "screen-name": "JedFiddler_540", "lang": "en", "friends-count": 11, "statuses-count": 264, "name": "Jed Fiddler", "followers-count": 48 }, "sender-location": point("44.11,91.45"), "send-time": datetime("2009-10-10T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " like at&t its voicemail-service is awesome:)", "countA": 220, "countB": 188 }
+{ "tweetid": 221i64, "user": { "screen-name": "RubinMueller_263", "lang": "en", "friends-count": 7, "statuses-count": 374, "name": "Rubin Mueller", "followers-count": 77 }, "sender-location": point("25.65,78.68"), "send-time": datetime("2006-09-20T10:10:00.000Z"), "referred-topics": {{ "sprint", "signal" }}, "message-text": " love sprint its signal is amazing", "countA": 221, "countB": 185 }
+{ "tweetid": 222i64, "user": { "screen-name": "JeniWeldi#255", "lang": "en", "friends-count": 71, "statuses-count": 259, "name": "Jeni Weldi", "followers-count": 63 }, "sender-location": point("27.21,67.74"), "send-time": datetime("2009-11-24T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " like iphone its customization is mind-blowing:)", "countA": 222, "countB": 51 }
+{ "tweetid": 223i64, "user": { "screen-name": "RochelleSaline@265", "lang": "en", "friends-count": 18, "statuses-count": 319, "name": "Rochelle Saline", "followers-count": 11 }, "sender-location": point("33.44,67.08"), "send-time": datetime("2007-10-23T10:10:00.000Z"), "referred-topics": {{ "iphone", "voice-command" }}, "message-text": " can't stand iphone its voice-command is OMG", "countA": 223, "countB": 84 }
+{ "tweetid": 224i64, "user": { "screen-name": "TabbyEckhardstein@204", "lang": "en", "friends-count": 5, "statuses-count": 207, "name": "Tabby Eckhardstein", "followers-count": 5 }, "sender-location": point("24.33,84.9"), "send-time": datetime("2005-02-01T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " hate samsung its speed is horrible", "countA": 224, "countB": 192 }
+{ "tweetid": 225i64, "user": { "screen-name": "FranklynBurns$23", "lang": "en", "friends-count": 73, "statuses-count": 475, "name": "Franklyn Burns", "followers-count": 131 }, "sender-location": point("26.5,94.14"), "send-time": datetime("2007-08-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " love motorola its reachability is good", "countA": 225, "countB": 65 }
+{ "tweetid": 226i64, "user": { "screen-name": "CorianderFischer#204", "lang": "en", "friends-count": 4, "statuses-count": 205, "name": "Coriander Fischer", "followers-count": 30 }, "sender-location": point("46.59,73.39"), "send-time": datetime("2010-03-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola its reachability is horrible", "countA": 226, "countB": 73 }
+{ "tweetid": 227i64, "user": { "screen-name": "EmikoBarth$856", "lang": "en", "friends-count": 28, "statuses-count": 7, "name": "Emiko Barth", "followers-count": 98 }, "sender-location": point("42.43,97.13"), "send-time": datetime("2005-03-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " can't stand verizon the reachability is bad:(", "countA": 227, "countB": 184 }
+{ "tweetid": 228i64, "user": { "screen-name": "LavoneStroh$831", "lang": "en", "friends-count": 53, "statuses-count": 2, "name": "Lavone Stroh", "followers-count": 44 }, "sender-location": point("32.04,66.17"), "send-time": datetime("2011-04-25T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " love t-mobile its touch-screen is awesome:)", "countA": 228, "countB": 38 }
+{ "tweetid": 229i64, "user": { "screen-name": "ShantelLaurence@745", "lang": "en", "friends-count": 78, "statuses-count": 195, "name": "Shantel Laurence", "followers-count": 124 }, "sender-location": point("46.58,97.58"), "send-time": datetime("2007-12-02T10:10:00.000Z"), "referred-topics": {{ "iphone", "wireless" }}, "message-text": " love iphone its wireless is awesome:)", "countA": 229, "countB": 26 }
+{ "tweetid": 230i64, "user": { "screen-name": "PiedadMosser@971", "lang": "en", "friends-count": 84, "statuses-count": 80, "name": "Piedad Mosser", "followers-count": 198 }, "sender-location": point("40.44,70.59"), "send-time": datetime("2005-06-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " hate motorola the wireless is bad", "countA": 230, "countB": 46 }
+{ "tweetid": 231i64, "user": { "screen-name": "CorrieHindman$963", "lang": "en", "friends-count": 54, "statuses-count": 78, "name": "Corrie Hindman", "followers-count": 180 }, "sender-location": point("26.04,77.36"), "send-time": datetime("2012-03-07T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " like sprint its wireless is mind-blowing:)", "countA": 231, "countB": 77 }
+{ "tweetid": 232i64, "user": { "screen-name": "DignaPorter_932", "lang": "en", "friends-count": 63, "statuses-count": 278, "name": "Digna Porter", "followers-count": 139 }, "sender-location": point("41.76,97.51"), "send-time": datetime("2009-12-01T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is mind-blowing:)", "countA": 232, "countB": 85 }
+{ "tweetid": 233i64, "user": { "screen-name": "TerrellStoddard@17", "lang": "en", "friends-count": 74, "statuses-count": 3, "name": "Terrell Stoddard", "followers-count": 35 }, "sender-location": point("34.89,73.83"), "send-time": datetime("2007-01-28T10:10:00.000Z"), "referred-topics": {{ "at&t", "touch-screen" }}, "message-text": " can't stand at&t the touch-screen is terrible", "countA": 233, "countB": 28 }
+{ "tweetid": 234i64, "user": { "screen-name": "EzraField#602", "lang": "en", "friends-count": 7, "statuses-count": 41, "name": "Ezra Field", "followers-count": 169 }, "sender-location": point("26.51,74.01"), "send-time": datetime("2011-03-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " love motorola the speed is good", "countA": 234, "countB": 94 }
+{ "tweetid": 235i64, "user": { "screen-name": "NathanielWentzel$505", "lang": "en", "friends-count": 40, "statuses-count": 178, "name": "Nathaniel Wentzel", "followers-count": 83 }, "sender-location": point("47.03,84.5"), "send-time": datetime("2007-07-02T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-clarity" }}, "message-text": " hate motorola its voice-clarity is bad", "countA": 235, "countB": 53 }
+{ "tweetid": 236i64, "user": { "screen-name": "AlanePycroft$112", "lang": "en", "friends-count": 61, "statuses-count": 222, "name": "Alane Pycroft", "followers-count": 125 }, "sender-location": point("31.82,81.86"), "send-time": datetime("2010-09-09T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is amazing:)", "countA": 236, "countB": 133 }
+{ "tweetid": 237i64, "user": { "screen-name": "EliseoMunson$584", "lang": "en", "friends-count": 21, "statuses-count": 353, "name": "Eliseo Munson", "followers-count": 9 }, "sender-location": point("39.12,74.86"), "send-time": datetime("2011-05-02T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " hate sprint its speed is terrible", "countA": 237, "countB": 100 }
+{ "tweetid": 238i64, "user": { "screen-name": "LupeCram@152", "lang": "en", "friends-count": 89, "statuses-count": 57, "name": "Lupe Cram", "followers-count": 51 }, "sender-location": point("40.34,71.58"), "send-time": datetime("2010-12-13T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is amazing", "countA": 238, "countB": 47 }
+{ "tweetid": 239i64, "user": { "screen-name": "AuroraChristman#544", "lang": "en", "friends-count": 96, "statuses-count": 435, "name": "Aurora Christman", "followers-count": 88 }, "sender-location": point("35.44,67.79"), "send-time": datetime("2009-08-11T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is mind-blowing", "countA": 239, "countB": 114 }
+{ "tweetid": 240i64, "user": { "screen-name": "JermainePotter_329", "lang": "en", "friends-count": 20, "statuses-count": 85, "name": "Jermaine Potter", "followers-count": 177 }, "sender-location": point("35.26,90.47"), "send-time": datetime("2012-04-17T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "network" }}, "message-text": " like t-mobile its network is mind-blowing:)", "countA": 240, "countB": 104 }
+{ "tweetid": 241i64, "user": { "screen-name": "KaetyHall$972", "lang": "en", "friends-count": 23, "statuses-count": 416, "name": "Kaety Hall", "followers-count": 87 }, "sender-location": point("25.06,93.28"), "send-time": datetime("2011-02-13T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " can't stand verizon its network is bad:(", "countA": 241, "countB": 45 }
+{ "tweetid": 242i64, "user": { "screen-name": "LaurenChristopher#195", "lang": "en", "friends-count": 12, "statuses-count": 149, "name": "Lauren Christopher", "followers-count": 183 }, "sender-location": point("24.03,79.65"), "send-time": datetime("2012-05-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " love t-mobile the touch-screen is amazing", "countA": 242, "countB": 179 }
+{ "tweetid": 243i64, "user": { "screen-name": "KoreyBonner$296", "lang": "en", "friends-count": 56, "statuses-count": 412, "name": "Korey Bonner", "followers-count": 190 }, "sender-location": point("42.54,97.81"), "send-time": datetime("2006-10-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "touch-screen" }}, "message-text": " like iphone its touch-screen is amazing:)", "countA": 243, "countB": 176 }
+{ "tweetid": 244i64, "user": { "screen-name": "LaetitiaWise@568", "lang": "en", "friends-count": 87, "statuses-count": 363, "name": "Laetitia Wise", "followers-count": 49 }, "sender-location": point("39.34,70.46"), "send-time": datetime("2010-11-25T10:10:00.000Z"), "referred-topics": {{ "iphone", "voicemail-service" }}, "message-text": " hate iphone its voicemail-service is terrible", "countA": 244, "countB": 150 }
+{ "tweetid": 245i64, "user": { "screen-name": "PatricaKellogg@123", "lang": "en", "friends-count": 4, "statuses-count": 80, "name": "Patrica Kellogg", "followers-count": 47 }, "sender-location": point("34.06,84.02"), "send-time": datetime("2005-05-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint its touch-screen is bad:(", "countA": 245, "countB": 83 }
+{ "tweetid": 246i64, "user": { "screen-name": "AdelleSwink$50", "lang": "en", "friends-count": 64, "statuses-count": 411, "name": "Adelle Swink", "followers-count": 165 }, "sender-location": point("28.66,71.42"), "send-time": datetime("2007-04-11T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " can't stand sprint the plan is horrible", "countA": 246, "countB": 128 }
+{ "tweetid": 247i64, "user": { "screen-name": "MckennaAlbright$534", "lang": "en", "friends-count": 94, "statuses-count": 374, "name": "Mckenna Albright", "followers-count": 61 }, "sender-location": point("45.26,93.62"), "send-time": datetime("2011-11-12T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " can't stand sprint the speed is OMG", "countA": 247, "countB": 194 }
+{ "tweetid": 248i64, "user": { "screen-name": "MerryReade$689", "lang": "en", "friends-count": 41, "statuses-count": 172, "name": "Merry Reade", "followers-count": 105 }, "sender-location": point("42.21,81.81"), "send-time": datetime("2009-08-18T10:10:00.000Z"), "referred-topics": {{ "verizon", "wireless" }}, "message-text": " like verizon its wireless is amazing", "countA": 248, "countB": 123 }
+{ "tweetid": 249i64, "user": { "screen-name": "CoryGoldvogel$187", "lang": "en", "friends-count": 91, "statuses-count": 359, "name": "Cory Goldvogel", "followers-count": 20 }, "sender-location": point("39.94,83.42"), "send-time": datetime("2005-09-11T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " dislike verizon its plan is bad:(", "countA": 249, "countB": 163 }
+{ "tweetid": 250i64, "user": { "screen-name": "RoystonRummel@500", "lang": "en", "friends-count": 34, "statuses-count": 154, "name": "Royston Rummel", "followers-count": 146 }, "sender-location": point("40.73,72.93"), "send-time": datetime("2005-11-20T10:10:00.000Z"), "referred-topics": {{ "samsung", "touch-screen" }}, "message-text": " love samsung its touch-screen is amazing:)", "countA": 250, "countB": 125 }
\ No newline at end of file
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
index f2a8438..ca049a6 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
@@ -315,7 +315,7 @@
mergePolicyFactory, mergePolicyFactoryProperties, new PrimaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
- storageProperties.getBloomFilterFalsePositiveRate(), true), false, searchCallbackFactory);
+ storageProperties.getBloomFilterFalsePositiveRate(), true), false, false, null, searchCallbackFactory);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
primaryPartitionConstraint);
return primarySearchOp;
@@ -402,7 +402,7 @@
}
StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(
new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.ScalarEvaluatorFactoryAdapter(selectCond),
- null, AqlBinaryBooleanInspectorImpl.FACTORY);
+ null, AqlBinaryBooleanInspectorImpl.FACTORY, false, -1, null);
AlgebricksMetaOperatorDescriptor asterixSelectOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 1,
new IPushRuntimeFactory[] { select }, new RecordDescriptor[] { secondaryRecDesc });
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, asterixSelectOp,
diff --git a/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql b/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql
new file mode 100644
index 0000000..25818a7
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql
@@ -0,0 +1,53 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
+
+write output to nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_01.adm";
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"count1":$t1.countA,
+"t2info": for $t2 in dataset('TweetMessages')
+ where $t1.countA /* +indexnl */= $t2.countB
+ order by $t2.tweetid
+ return {"tweetid2": $t2.tweetid,
+ "count2":$t2.countB}
+};
diff --git a/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02.aql b/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02.aql
new file mode 100644
index 0000000..70a31e1
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02.aql
@@ -0,0 +1,55 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
+
+write output to nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_02.adm";
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"count1":$t1.countA,
+"t2info": for $t2 in dataset('TweetMessages')
+ where $t1.countA /* +indexnl */= $t2.countB and
+ $t1.tweetid != $t2.tweetid
+ order by $t2.tweetid
+ return {"tweetid2": $t2.tweetid,
+ "count2":$t2.countB}
+};
+
diff --git a/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/issue741.aql b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/issue741.aql
new file mode 100644
index 0000000..7402ec9
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/issue741.aql
@@ -0,0 +1,47 @@
+/*
+ * Description : Test that left-outer-join may use an available inverted index in index subtree.
+ * Issue : 741
+ * Expected Res : Success
+ * Date : 16th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as {
+screen_name: string,
+lang: string,
+friends_count: int32,
+statuses_count: int32,
+name: string,
+followers_count: int32
+}
+
+create type TweetMessageType as {
+tweetid: int64,
+user: TwitterUserType,
+sender_location: point?,
+send_time: datetime,
+referred_topics: {{ string }},
+message_text: string
+}
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create index topicIIx on TweetMessages(referred_topics) type keyword;
+
+write output to nc1:"rttest/inverted-index-join_issue741.adm";
+
+for $t in dataset('TweetMessages')
+where $t.send_time >= datetime('2011-06-18T14:10:17')
+and
+$t.send_time < datetime('2011-06-18T15:10:17')
+return {
+ "tweet": $t.tweetid,
+ "similar-tweets": for $t2 in dataset('TweetMessages')
+ let $sim := similarity-jaccard-check($t.referred_topics, $t2.referred_topics, 0.6f)
+ where $sim[0] and
+ $t2.tweetid != $t.tweetid
+ return $t2.tweetid
+}
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.aql b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.aql
new file mode 100644
index 0000000..7adbf3a
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.aql
@@ -0,0 +1,55 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgKeywordIx on TweetMessages(message-text) type keyword;
+create index msgNgramIx on TweetMessages(message-text) type ngram(3);
+create index topicKeywordIx on TweetMessages(referred-topics) type keyword;
+
+write output to nc1:"rttest/inverted-index-join_leftouterjoin-probe-pidx-with-join-edit-distance-check_idx_01.adm";
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid > int64("240")
+order by $t1.tweetid
+return {
+ "tweet": {"id": $t1.tweetid, "topics" : $t1.message-text} ,
+ "similar-tweets": for $t2 in dataset('TweetMessages')
+ let $sim := edit-distance-check($t1.message-text, $t2.message-text, 7)
+ where $sim[0] and
+ $t2.tweetid != $t1.tweetid
+ order by $t2.tweetid
+ return {"id": $t2.tweetid, "topics" : $t2.message-text}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.aql b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.aql
new file mode 100644
index 0000000..b7cd179
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.aql
@@ -0,0 +1,55 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgKeywordIx on TweetMessages(message-text) type keyword;
+create index msgNgramIx on TweetMessages(message-text) type ngram(3);
+create index topicKeywordIx on TweetMessages(referred-topics) type keyword;
+
+write output to nc1:"rttest/inverted-index-join_leftouterjoin-probe-pidx-with-join-jaccard-check_idx_01.adm";
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid > int64("240")
+order by $t1.tweetid
+return {
+ "tweet": {"id": $t1.tweetid, "topics" : $t1.referred-topics} ,
+ "similar-tweets": for $t2 in dataset('TweetMessages')
+ let $sim := similarity-jaccard-check($t1.referred-topics, $t2.referred-topics, 0.5f)
+ where $sim[0] and
+ $t2.tweetid != $t1.tweetid
+ order by $t2.tweetid
+ return {"id": $t2.tweetid, "topics" : $t2.referred-topics}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-contains.aql b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-contains.aql
new file mode 100644
index 0000000..676ed3b
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-contains.aql
@@ -0,0 +1,47 @@
+/*
+ * Description : Tests whether an ngram_index is applied to optimize a join query using the contains function.
+ * The index should be applied.
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP(title) type ngram(3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-contains.adm";
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP(title) type ngram(3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-contains.adm";
+
+for $o1 in dataset('DBLP')
+for $o2 in dataset('DBLP')
+where contains($o1.title, $o2.title) and $o1.id < $o2.id
+order by $o1.id, $o2.id
+return {"title1":$o1.title, "title2":$o2.title}
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard-check-after-btree-access.aql b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard-check-after-btree-access.aql
index d656045..f58f6bc 100644
--- a/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard-check-after-btree-access.aql
+++ b/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard-check-after-btree-access.aql
@@ -42,7 +42,7 @@
for $t1 in dataset('TweetMessages')
for $t2 in dataset('TweetMessages')
-let $sim := similarity-jaccard-check($t1.message-text, $t2.message-text, 0.6f)
+let $sim := similarity-jaccard-check(word-tokens($t1.message-text), word-tokens($t2.message-text), 0.6f)
where $sim[0] and $t1.tweetid < int64("20") and $t2.tweetid != $t1.tweetid
return {
"t1": $t1.tweetid,
diff --git a/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/issue730.aql b/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/issue730.aql
new file mode 100644
index 0000000..e67d702
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/issue730.aql
@@ -0,0 +1,44 @@
+/*
+ * Description : Test that left-outer-join may use an available rtree index in index subtree.
+ * Issue : 730
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as {
+screen_name: string,
+lang: string,
+friends_count: int32,
+statuses_count: int32,
+name: string,
+followers_count: int32
+}
+
+create type TweetMessageType as {
+tweetid: int64,
+user: TwitterUserType,
+sender_location: point?,
+send_time: datetime,
+referred_topics: {{ string }},
+message_text: string
+}
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender_location) type rtree;
+
+write output to nc1:"rttest/rtree-index-join_issue730.adm";
+
+for $t1 in dataset('TweetMessages')
+where $t1.send_time >= datetime('2011-06-18T14:10:17') and $t1.send_time < datetime('2011-06-18T15:10:17')
+let $n := create-circle($t1.sender_location, 5.0)
+return {
+"message": $t1.tweetid,
+"nearby-message": for $t2 in dataset('TweetMessages')
+ where spatial-intersect($t2.sender_location, $n)
+ return $t2.tweetid
+}
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_01.aql b/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_01.aql
new file mode 100644
index 0000000..9b72850
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_01.aql
@@ -0,0 +1,53 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
+
+write output to nc1:"rttest/rtree-index-join_leftouterjoin-probe-pidx-with-join-rtree-sidx_01.adm";
+
+for $t1 in dataset('TweetMessages')
+let $n := create-circle($t1.sender-location, 0.5)
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"loc1":$t1.sender-location,
+"nearby-message": for $t2 in dataset('TweetMessages')
+ where spatial-intersect($t2.sender-location, $n)
+ order by $t2.tweetid
+ return {"tweetid2":$t2.tweetid, "loc2":$t2.sender-location}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_02.aql b/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_02.aql
new file mode 100644
index 0000000..d71faad
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_02.aql
@@ -0,0 +1,53 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
+
+write output to nc1:"rttest/rtree-index-join_leftouterjoin-probe-pidx-with-join-rtree-sidx_02.adm";
+
+for $t1 in dataset('TweetMessages')
+let $n := create-circle($t1.sender-location, 0.5)
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"loc1":$t1.sender-location,
+"nearby-message": for $t2 in dataset('TweetMessages')
+ where spatial-intersect($t2.sender-location, $n) and $t1.tweetid != $t2.tweetid
+ order by $t2.tweetid
+ return {"tweetid2":$t2.tweetid, "loc2":$t2.sender-location}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.plan b/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.plan
new file mode 100644
index 0000000..ddcbf08
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.plan
@@ -0,0 +1,38 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$28(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$28(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$24] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC), $$21(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$24] |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$36(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02.plan b/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02.plan
new file mode 100644
index 0000000..48f4d02
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02.plan
@@ -0,0 +1,38 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$33(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$33(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$29] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$29(ASC), $$25(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$29] |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$41(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/issue741.plan b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/issue741.plan
new file mode 100644
index 0000000..55a81d2
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/issue741.plan
@@ -0,0 +1,46 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$29] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$29(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$29] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$37][$$29] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$29] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$40(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.plan b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.plan
new file mode 100644
index 0000000..c132181
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.plan
@@ -0,0 +1,78 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$36(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$36(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$32] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC), $$28(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$32] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$45][$$32] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$32] |PARTITIONED|
+ -- UNION_ALL |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$52(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.plan b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.plan
new file mode 100644
index 0000000..3252899
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.plan
@@ -0,0 +1,47 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$36(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$36(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$32] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC), $$28(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$32] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$45][$$32] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$32] |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$48(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-contains.plan b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-contains.plan
new file mode 100644
index 0000000..0d030ef
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-contains.plan
@@ -0,0 +1,25 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$18(ASC), $$19(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$18(ASC), $$19(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$28(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan
index 237864f..c366d1b 100644
--- a/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan
+++ b/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan
@@ -4,31 +4,34 @@
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$34][$$22] |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$37][$$24] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$22] |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$37(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$24] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$41(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/issue730.plan b/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/issue730.plan
new file mode 100644
index 0000000..40582f1
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/issue730.plan
@@ -0,0 +1,37 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$24] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$24] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$37(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_01.plan b/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_01.plan
new file mode 100644
index 0000000..e0480a2
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_01.plan
@@ -0,0 +1,41 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$30(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$27] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$27(ASC), $$24(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$27] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$43(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_02.plan b/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_02.plan
new file mode 100644
index 0000000..c6c7d74
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/rtree-index-join/leftouterjoin-probe-pidx-with-join-rtree-sidx_02.plan
@@ -0,0 +1,41 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$35(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$35(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$32] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$32(ASC), $$28(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$32] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$48(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.ddl.aql
new file mode 100644
index 0000000..8aa7b64
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.ddl.aql
@@ -0,0 +1,38 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.2.update.aql
new file mode 100644
index 0000000..51d3641
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+load dataset TweetMessages
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/twitter/tw_for_indexleftouterjoin.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.3.query.aql
new file mode 100644
index 0000000..2490956
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.3.query.aql
@@ -0,0 +1,21 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"count1":$t1.countA,
+"t2info": for $t2 in dataset('TweetMessages')
+ where $t1.countA /* +indexnl */= $t2.countB
+ order by $t2.tweetid
+ return {"tweetid2": $t2.tweetid,
+ "count2":$t2.countB}
+};
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.1.ddl.aql
new file mode 100644
index 0000000..8aa7b64
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.1.ddl.aql
@@ -0,0 +1,38 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.2.update.aql
new file mode 100644
index 0000000..51d3641
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+load dataset TweetMessages
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/twitter/tw_for_indexleftouterjoin.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.3.query.aql
new file mode 100644
index 0000000..956aecf
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.3.query.aql
@@ -0,0 +1,22 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"count1":$t1.countA,
+"t2info": for $t2 in dataset('TweetMessages')
+ where $t1.countA /* +indexnl */= $t2.countB and
+ $t1.tweetid != $t2.tweetid
+ order by $t2.tweetid
+ return {"tweetid2": $t2.tweetid,
+ "count2":$t2.countB}
+};
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.1.ddl.aql
new file mode 100644
index 0000000..4d83362
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.1.ddl.aql
@@ -0,0 +1,40 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgKeywordIx on TweetMessages(message-text) type keyword;
+create index msgNgramIx on TweetMessages(message-text) type ngram(3);
+create index topicKeywordIx on TweetMessages(referred-topics) type keyword;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.2.update.aql
new file mode 100644
index 0000000..51d3641
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+load dataset TweetMessages
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/twitter/tw_for_indexleftouterjoin.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.3.query.aql
new file mode 100644
index 0000000..e21cf7a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.3.query.aql
@@ -0,0 +1,21 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary keyword inverted index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 16th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid > int64("240")
+order by $t1.tweetid
+return {
+ "tweet": {"id": $t1.tweetid, "topics" : $t1.referred-topics} ,
+ "similar-tweets": for $t2 in dataset('TweetMessages')
+ let $sim := similarity-jaccard-check($t1.referred-topics, $t2.referred-topics, 0.5f)
+ where $sim[0] and
+ $t2.tweetid != $t1.tweetid
+ order by $t2.tweetid
+ return {"id": $t2.tweetid, "topics" : $t2.referred-topics}
+};
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.1.ddl.aql
new file mode 100644
index 0000000..4d83362
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.1.ddl.aql
@@ -0,0 +1,40 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgKeywordIx on TweetMessages(message-text) type keyword;
+create index msgNgramIx on TweetMessages(message-text) type ngram(3);
+create index topicKeywordIx on TweetMessages(referred-topics) type keyword;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.2.update.aql
new file mode 100644
index 0000000..51d3641
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+load dataset TweetMessages
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/twitter/tw_for_indexleftouterjoin.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.3.query.aql
new file mode 100644
index 0000000..01cf073
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx2.3.query.aql
@@ -0,0 +1,21 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary keyword inverted index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 16th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid > int64("240")
+order by $t1.tweetid
+return {
+ "tweet": {"id": $t1.tweetid, "topics" : $t1.message-text} ,
+ "similar-tweets": for $t2 in dataset('TweetMessages')
+ let $sim := edit-distance-check($t1.message-text, $t2.message-text, 7)
+ where $sim[0] and
+ $t2.tweetid != $t1.tweetid
+ order by $t2.tweetid
+ return {"id": $t2.tweetid, "topics" : $t2.message-text}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.1.ddl.aql
new file mode 100644
index 0000000..a353da2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.1.ddl.aql
@@ -0,0 +1,38 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.2.update.aql
new file mode 100644
index 0000000..4763f3c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+load dataset TweetMessages
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/twitter/tw_for_indexleftouterjoin.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.3.query.aql
new file mode 100644
index 0000000..50e72e3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.3.query.aql
@@ -0,0 +1,21 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+let $n := create-circle($t1.sender-location, 0.5)
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"loc1":$t1.sender-location,
+"nearby-message": for $t2 in dataset('TweetMessages')
+ where spatial-intersect($t2.sender-location, $n)
+ order by $t2.tweetid
+ return {"tweetid2":$t2.tweetid, "loc2":$t2.sender-location}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.1.ddl.aql
new file mode 100644
index 0000000..a353da2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.1.ddl.aql
@@ -0,0 +1,38 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.2.update.aql
new file mode 100644
index 0000000..4763f3c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+load dataset TweetMessages
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/twitter/tw_for_indexleftouterjoin.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.3.query.aql
new file mode 100644
index 0000000..1218e03
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.3.query.aql
@@ -0,0 +1,21 @@
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+let $n := create-circle($t1.sender-location, 0.5)
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"loc1":$t1.sender-location,
+"nearby-message": for $t2 in dataset('TweetMessages')
+ where spatial-intersect($t2.sender-location, $n) and $t1.tweetid != $t2.tweetid
+ order by $t2.tweetid
+ return {"tweetid2":$t2.tweetid, "loc2":$t2.sender-location}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.adm b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.adm
new file mode 100644
index 0000000..1907bca
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx1/probe-pidx-with-join-btree-sidx1.1.adm
@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "count1": 1, "t2info": [ ] }
+{ "tweetid1": 2i64, "count1": 2, "t2info": [ { "tweetid2": 60i64, "count2": 2 } ] }
+{ "tweetid1": 3i64, "count1": 3, "t2info": [ { "tweetid2": 105i64, "count2": 3 }, { "tweetid2": 206i64, "count2": 3 } ] }
+{ "tweetid1": 4i64, "count1": 4, "t2info": [ ] }
+{ "tweetid1": 5i64, "count1": 5, "t2info": [ { "tweetid2": 138i64, "count2": 5 }, { "tweetid2": 175i64, "count2": 5 } ] }
+{ "tweetid1": 6i64, "count1": 6, "t2info": [ { "tweetid2": 148i64, "count2": 6 } ] }
+{ "tweetid1": 7i64, "count1": 7, "t2info": [ { "tweetid2": 125i64, "count2": 7 } ] }
+{ "tweetid1": 8i64, "count1": 8, "t2info": [ ] }
+{ "tweetid1": 9i64, "count1": 9, "t2info": [ { "tweetid2": 141i64, "count2": 9 } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.1.adm b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.1.adm
new file mode 100644
index 0000000..1907bca
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-btree-sidx2/probe-pidx-with-join-btree-sidx2.1.adm
@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "count1": 1, "t2info": [ ] }
+{ "tweetid1": 2i64, "count1": 2, "t2info": [ { "tweetid2": 60i64, "count2": 2 } ] }
+{ "tweetid1": 3i64, "count1": 3, "t2info": [ { "tweetid2": 105i64, "count2": 3 }, { "tweetid2": 206i64, "count2": 3 } ] }
+{ "tweetid1": 4i64, "count1": 4, "t2info": [ ] }
+{ "tweetid1": 5i64, "count1": 5, "t2info": [ { "tweetid2": 138i64, "count2": 5 }, { "tweetid2": 175i64, "count2": 5 } ] }
+{ "tweetid1": 6i64, "count1": 6, "t2info": [ { "tweetid2": 148i64, "count2": 6 } ] }
+{ "tweetid1": 7i64, "count1": 7, "t2info": [ { "tweetid2": 125i64, "count2": 7 } ] }
+{ "tweetid1": 8i64, "count1": 8, "t2info": [ ] }
+{ "tweetid1": 9i64, "count1": 9, "t2info": [ { "tweetid2": 141i64, "count2": 9 } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.1.adm b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.1.adm
new file mode 100644
index 0000000..d7b26bc
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-invidx-sidx1/probe-pidx-with-join-invidx-sidx1.1.adm
@@ -0,0 +1,10 @@
+{ "tweet": { "id": 241i64, "topics": {{ "verizon", "network" }} }, "similar-tweets": [ { "id": 40i64, "topics": {{ "verizon", "network" }} }, { "id": 45i64, "topics": {{ "verizon", "network" }} }, { "id": 46i64, "topics": {{ "verizon", "network" }} }, { "id": 112i64, "topics": {{ "verizon", "network" }} } ] }
+{ "tweet": { "id": 242i64, "topics": {{ "t-mobile", "touch-screen" }} }, "similar-tweets": [ { "id": 47i64, "topics": {{ "t-mobile", "touch-screen" }} }, { "id": 150i64, "topics": {{ "t-mobile", "touch-screen" }} }, { "id": 228i64, "topics": {{ "t-mobile", "touch-screen" }} } ] }
+{ "tweet": { "id": 243i64, "topics": {{ "iphone", "touch-screen" }} }, "similar-tweets": [ { "id": 186i64, "topics": {{ "iphone", "touch-screen" }} } ] }
+{ "tweet": { "id": 244i64, "topics": {{ "iphone", "voicemail-service" }} }, "similar-tweets": [ { "id": 184i64, "topics": {{ "iphone", "voicemail-service" }} } ] }
+{ "tweet": { "id": 245i64, "topics": {{ "sprint", "touch-screen" }} }, "similar-tweets": [ { "id": 60i64, "topics": {{ "sprint", "touch-screen" }} }, { "id": 168i64, "topics": {{ "sprint", "touch-screen" }} }, { "id": 196i64, "topics": {{ "sprint", "touch-screen" }} } ] }
+{ "tweet": { "id": 246i64, "topics": {{ "sprint", "plan" }} }, "similar-tweets": [ { "id": 49i64, "topics": {{ "sprint", "plan" }} }, { "id": 130i64, "topics": {{ "sprint", "plan" }} } ] }
+{ "tweet": { "id": 247i64, "topics": {{ "sprint", "speed" }} }, "similar-tweets": [ { "id": 59i64, "topics": {{ "sprint", "speed" }} }, { "id": 208i64, "topics": {{ "sprint", "speed" }} }, { "id": 237i64, "topics": {{ "sprint", "speed" }} } ] }
+{ "tweet": { "id": 248i64, "topics": {{ "verizon", "wireless" }} }, "similar-tweets": [ ] }
+{ "tweet": { "id": 249i64, "topics": {{ "verizon", "plan" }} }, "similar-tweets": [ { "id": 179i64, "topics": {{ "verizon", "plan" }} }, { "id": 181i64, "topics": {{ "verizon", "plan" }} }, { "id": 212i64, "topics": {{ "verizon", "plan" }} } ] }
+{ "tweet": { "id": 250i64, "topics": {{ "samsung", "touch-screen" }} }, "similar-tweets": [ { "id": 124i64, "topics": {{ "samsung", "touch-screen" }} } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx1.1.adm b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx1.1.adm
new file mode 100644
index 0000000..188c292
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-invidx-sidx2/probe-pidx-with-join-invidx-sidx1.1.adm
@@ -0,0 +1,10 @@
+{ "tweet": { "id": 241i64, "topics": " can't stand verizon its network is bad:(" }, "similar-tweets": [ { "id": 112i64, "topics": " can't stand verizon its network is terrible:(" } ] }
+{ "tweet": { "id": 242i64, "topics": " love t-mobile the touch-screen is amazing" }, "similar-tweets": [ ] }
+{ "tweet": { "id": 243i64, "topics": " like iphone its touch-screen is amazing:)" }, "similar-tweets": [ ] }
+{ "tweet": { "id": 244i64, "topics": " hate iphone its voicemail-service is terrible" }, "similar-tweets": [ ] }
+{ "tweet": { "id": 245i64, "topics": " hate sprint its touch-screen is bad:(" }, "similar-tweets": [ { "id": 60i64, "topics": " hate sprint its touch-screen is OMG:(" }, { "id": 196i64, "topics": " hate sprint the touch-screen is OMG:(" } ] }
+{ "tweet": { "id": 246i64, "topics": " can't stand sprint the plan is horrible" }, "similar-tweets": [ ] }
+{ "tweet": { "id": 247i64, "topics": " can't stand sprint the speed is OMG" }, "similar-tweets": [ ] }
+{ "tweet": { "id": 248i64, "topics": " like verizon its wireless is amazing" }, "similar-tweets": [ ] }
+{ "tweet": { "id": 249i64, "topics": " dislike verizon its plan is bad:(" }, "similar-tweets": [ ] }
+{ "tweet": { "id": 250i64, "topics": " love samsung its touch-screen is amazing:)" }, "similar-tweets": [ ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.1.adm b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.1.adm
new file mode 100644
index 0000000..b4337b3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-rtree-sidx1/probe-pidx-with-join-rtree-sidx1.1.adm
@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "loc1": point("42.83,72.44"), "nearby-message": [ { "tweetid2": 1i64, "loc2": point("42.83,72.44") }, { "tweetid2": 55i64, "loc2": point("42.77,72.16") }, { "tweetid2": 114i64, "loc2": point("42.87,72.38") } ] }
+{ "tweetid1": 2i64, "loc1": point("34.81,72.44"), "nearby-message": [ { "tweetid2": 2i64, "loc2": point("34.81,72.44") } ] }
+{ "tweetid1": 3i64, "loc1": point("24.54,82.66"), "nearby-message": [ { "tweetid2": 3i64, "loc2": point("24.54,82.66") } ] }
+{ "tweetid1": 4i64, "loc1": point("38.14,68.1"), "nearby-message": [ { "tweetid2": 4i64, "loc2": point("38.14,68.1") } ] }
+{ "tweetid1": 5i64, "loc1": point("35.4,68.89"), "nearby-message": [ { "tweetid2": 5i64, "loc2": point("35.4,68.89") } ] }
+{ "tweetid1": 6i64, "loc1": point("42.75,78.5"), "nearby-message": [ { "tweetid2": 6i64, "loc2": point("42.75,78.5") } ] }
+{ "tweetid1": 7i64, "loc1": point("48.16,71.59"), "nearby-message": [ { "tweetid2": 7i64, "loc2": point("48.16,71.59") }, { "tweetid2": 42i64, "loc2": point("47.86,71.93") }, { "tweetid2": 192i64, "loc2": point("48.12,72.0") } ] }
+{ "tweetid1": 8i64, "loc1": point("36.17,72.56"), "nearby-message": [ { "tweetid2": 8i64, "loc2": point("36.17,72.56") } ] }
+{ "tweetid1": 9i64, "loc1": point("38.02,70.38"), "nearby-message": [ { "tweetid2": 9i64, "loc2": point("38.02,70.38") }, { "tweetid2": 51i64, "loc2": point("37.65,70.54") } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.1.adm b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.1.adm
new file mode 100644
index 0000000..826e20a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-leftouterjoin/probe-pidx-with-join-rtree-sidx2/probe-pidx-with-join-rtree-sidx2.1.adm
@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "loc1": point("42.83,72.44"), "nearby-message": [ { "tweetid2": 55i64, "loc2": point("42.77,72.16") }, { "tweetid2": 114i64, "loc2": point("42.87,72.38") } ] }
+{ "tweetid1": 2i64, "loc1": point("34.81,72.44"), "nearby-message": [ ] }
+{ "tweetid1": 3i64, "loc1": point("24.54,82.66"), "nearby-message": [ ] }
+{ "tweetid1": 4i64, "loc1": point("38.14,68.1"), "nearby-message": [ ] }
+{ "tweetid1": 5i64, "loc1": point("35.4,68.89"), "nearby-message": [ ] }
+{ "tweetid1": 6i64, "loc1": point("42.75,78.5"), "nearby-message": [ ] }
+{ "tweetid1": 7i64, "loc1": point("48.16,71.59"), "nearby-message": [ { "tweetid2": 42i64, "loc2": point("47.86,71.93") }, { "tweetid2": 192i64, "loc2": point("48.12,72.0") } ] }
+{ "tweetid1": 8i64, "loc1": point("36.17,72.56"), "nearby-message": [ ] }
+{ "tweetid1": 9i64, "loc1": point("38.02,70.38"), "nearby-message": [ { "tweetid2": 51i64, "loc2": point("37.65,70.54") } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 94fa78c..9c62816 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -5000,6 +5000,38 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="index-leftouterjoin">
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-btree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-btree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-rtree-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-rtree-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx1">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-leftouterjoin">
+ <compilation-unit name="probe-pidx-with-join-invidx-sidx2">
+ <output-dir compare="Text">probe-pidx-with-join-invidx-sidx2</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="distinct">
<test-case FilePath="distinct">
<compilation-unit name="query-issue443">
diff --git a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
index ee94c33..6f2e0a0 100644
--- a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
+++ b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
@@ -437,11 +437,11 @@
qarFile.getParentFile().mkdirs();
TestsUtils.writeResultsToFile(qarFile, resultStream);
- TestsUtils.runScriptAndCompareWithResult(testFile, new PrintWriter(System.err),
- qbcFile, qarFile);
+ TestsUtils.runScriptAndCompareWithResult(testFile, new PrintWriter(System.err), qbcFile,
+ qarFile);
- LOGGER.info("[TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/"
- + cUnit.getName() + " PASSED ");
+ LOGGER.info("[TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName()
+ + " PASSED ");
break;
case "txneu": //eu represents erroneous update
try {
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 78481fa..fc73854 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -311,7 +311,7 @@
String dataverseName = asid.getDataverseName();
String datasetName = asid.getDatasetName();
Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, datasetName);
- return buildBtreeRuntime(jobSpec, outputVars, opSchema, typeEnv, context, true,
+ return buildBtreeRuntime(jobSpec, outputVars, opSchema, typeEnv, context, true, false,
((DatasetDataSource) dataSource).getDataset(), primaryIndex.getIndexName(), null, null, true, true,
implConfig);
}
@@ -489,9 +489,9 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildBtreeRuntime(JobSpecification jobSpec,
List<LogicalVariable> outputVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
- JobGenContext context, boolean retainInput, Dataset dataset, String indexName, int[] lowKeyFields,
- int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive, Object implConfig)
- throws AlgebricksException {
+ JobGenContext context, boolean retainInput, boolean retainNull, Dataset dataset, String indexName,
+ int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
+ Object implConfig) throws AlgebricksException {
boolean isSecondary = true;
try {
Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
@@ -571,7 +571,7 @@
dataset.getDatasetId()) : new PrimaryIndexOperationTrackerProvider(
dataset.getDatasetId()), rtcProvider, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), !isSecondary), retainInput,
- searchCallbackFactory);
+ retainNull, context.getNullWriterFactory(), searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeSearchOp, spPc.second);
@@ -582,8 +582,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildRtreeRuntime(JobSpecification jobSpec,
List<LogicalVariable> outputVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
- JobGenContext context, boolean retainInput, Dataset dataset, String indexName, int[] keyFields)
- throws AlgebricksException {
+ JobGenContext context, boolean retainInput, boolean retainNull, Dataset dataset, String indexName,
+ int[] keyFields) throws AlgebricksException {
try {
ARecordType recType = (ARecordType) findType(dataset.getDataverseName(), dataset.getItemTypeName());
int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
@@ -647,7 +647,8 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMRTreeIOOperationCallbackFactory.INSTANCE, proposeLinearizer(nestedKeyType.getTypeTag(),
comparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
- btreeFields), retainInput, searchCallbackFactory);
+ btreeFields), retainInput, retainNull, context.getNullWriterFactory(),
+ searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeSearchOp, spPc.second);
} catch (MetadataException me) {
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
index 9e03007..77f758b 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
@@ -18,6 +18,8 @@
import java.io.IOException;
import java.util.Arrays;
+import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.ListItemBinaryComparatorFactory;
+import edu.uci.ics.asterix.dataflow.data.nontagged.hash.ListItemBinaryHashFunctionFactory;
import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import edu.uci.ics.asterix.om.base.AFloat;
import edu.uci.ics.asterix.om.base.AMutableFloat;
@@ -36,8 +38,6 @@
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.ListItemBinaryComparatorFactory;
-import edu.uci.ics.asterix.dataflow.data.nontagged.hash.ListItemBinaryHashFunctionFactory;
public class SimilarityJaccardEvaluator implements ICopyEvaluator {
@@ -120,6 +120,11 @@
firstTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[firstStart]);
secondTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[secondStart]);
+ if (firstTypeTag == ATypeTag.NULL)
+ return;
+ if (secondTypeTag == ATypeTag.NULL)
+ return;
+
firstItemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[firstStart + 1]);
secondItemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[secondStart + 1]);
}