[ASTERIXDB-3303][COMP] Projection Sizes continued

Change-Id: I5a839f23232707a7c2a07ccded858e5e589ae23e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17970
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Vijay Sarathy <vijay.sarathy@couchbase.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
index 510d8bb..d9174d8 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
@@ -38,7 +38,6 @@
 import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.metadata.declared.SampleDataSource;
 import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.om.base.AInt64;
 import org.apache.asterix.om.base.AOrderedList;
 import org.apache.asterix.om.base.IAObject;
 import org.apache.asterix.om.constants.AsterixConstantValue;
@@ -915,7 +914,7 @@
                 // There are predicates here. So skip the predicates and get the original dataset card.
                 // Now apply all the predicates and get the card after all predicates are applied.
                 result = stats.runSamplingQueryProjection(this.optCtx, leafInput);
-                double predicateCardinality = stats.findPredicateCardinality(result);
+                double predicateCardinality = stats.findPredicateCardinality(result, true);
 
                 double projectedSize;
                 if (predicateCardinality > 0.0) { // otherwise, we get nulls for the averages
@@ -924,7 +923,7 @@
                     ILogicalExpression saveExpr = selop.getCondition().getValue();
                     selop.getCondition().setValue(ConstantExpression.TRUE);
                     result = stats.runSamplingQueryProjection(this.optCtx, leafInput);
-                    double x = stats.findPredicateCardinality(result);
+                    double x = stats.findPredicateCardinality(result, true);
                     // better to check if x is 0
                     if (x == 0.0) {
                         int fields = stats.numberOfFields(result);
@@ -1087,7 +1086,7 @@
                 SelectOperator selOp = new SelectOperator(new MutableObject<>(exp));
                 selOp.getInputs().add(new MutableObject<>(leafInput));
                 result = stats.runSamplingQuery(this.optCtx, selOp);
-                predicateCardinality = (double) ((AInt64) result.get(0).get(0)).getLongValue();
+                predicateCardinality = stats.findPredicateCardinality(result, false);
 
                 if (predicateCardinality == 0.0) {
                     predicateCardinality = 0.0001 * idxDetails.getSampleCardinalityTarget();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
index 9615386..b6bcd43 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
@@ -505,7 +505,7 @@
             }
         }
 
-        double predicateCardinality = (double) ((AInt64) result.get(0).get(0)).getLongValue();
+        double predicateCardinality = findPredicateCardinality(result, false);
         if (predicateCardinality == 0.0) {
             predicateCardinality = 0.0001 * idxDetails.getSampleCardinalityTarget();
         }
@@ -528,7 +528,7 @@
             selOp.getCondition().setValue(ConstantExpression.TRUE);
             result = runSamplingQuery(optCtx, selOp);
             selOp.getCondition().setValue(saveExprs);
-            sampleCard = (double) ((AInt64) result.get(0).get(0)).getLongValue();
+            sampleCard = findPredicateCardinality(result, false);
         }
         // switch  the scanOp back
         parent.getInputs().get(0).setValue(scanOp);
@@ -537,12 +537,15 @@
         return sel;
     }
 
-    public double findPredicateCardinality(List<List<IAObject>> result) {
-        ARecord record = (ARecord) (((IAObject) ((List<IAObject>) (result.get(0))).get(0)));
-        int fields = record.numberOfFields();
-        IAObject first = record.getValueByPos(0);
-        double predicateCardinality = ((double) ((AInt64) first).getLongValue());
-        return predicateCardinality;
+    public double findPredicateCardinality(List<List<IAObject>> result, boolean project) {
+        if (project) {
+            ARecord record = (ARecord) (((IAObject) ((List<IAObject>) (result.get(0))).get(0)));
+            int fields = record.numberOfFields();
+            IAObject first = record.getValueByPos(0);
+            return ((double) ((AInt64) first).getLongValue());
+        } else {
+            return (double) ((AInt64) result.get(0).get(0)).getLongValue();
+        }
     }
 
     public int numberOfFields(List<List<IAObject>> result) {
@@ -763,7 +766,7 @@
         ILogicalOperator copyOfSelOp = OperatorManipulationUtil.bottomUpCopyOperators(selOp);
         if (setSampleDataSource(copyOfSelOp, sampleDataSource)) {
             List<List<IAObject>> result = runSamplingQuery(optCtx, copyOfSelOp);
-            sampleSize = (long) ((AInt64) result.get(0).get(0)).getLongValue();
+            sampleSize = (long) findPredicateCardinality(result, false);
         }
         return sampleSize;
     }
@@ -779,7 +782,7 @@
             if (setSampleDataSource(copyOfGrpByDistinctOp, sampleDataSource)) {
                 // get distinct cardinality from the sampling source
                 List<List<IAObject>> result = runSamplingQuery(optCtx, copyOfGrpByDistinctOp);
-                estDistCardinalityFromSample = (double) ((AInt64) result.get(0).get(0)).getLongValue();
+                estDistCardinalityFromSample = findPredicateCardinality(result, false);
             }
         }
         if (estDistCardinalityFromSample != -1.0) { // estimate distinct cardinality for the dataset from the sampled cardinality