[NO ISSUE][COMP] Meta() after group by
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Using meta() after group-by requires meta() be assigned to
a variable in a LET clause to expose meta().
- added source location to IFunctionManager.lookupFunction()
- added test cases
Change-Id: I2b173e23352fa80ef57154f08910586c6ee196b8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3528
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
index 32f8001..0de905f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
@@ -52,6 +52,7 @@
import org.apache.hyracks.algebricks.runtime.base.ISerializedAggregateEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
+import org.apache.hyracks.api.exceptions.SourceLocation;
public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
@@ -197,8 +198,9 @@
private IFunctionDescriptor resolveFunction(AbstractFunctionCallExpression expr, IVariableTypeEnvironment env,
JobGenContext context) throws AlgebricksException {
FunctionIdentifier fnId = expr.getFunctionIdentifier();
- IFunctionDescriptor fd = functionManager.lookupFunction(fnId);
- fd.setSourceLocation(expr.getSourceLocation());
+ SourceLocation sourceLocation = expr.getSourceLocation();
+ IFunctionDescriptor fd = functionManager.lookupFunction(fnId, sourceLocation);
+ fd.setSourceLocation(sourceLocation);
IFunctionTypeInferer fnTypeInfer = functionManager.lookupFunctionTypeInferer(fnId);
if (fnTypeInfer != null) {
CompilerProperties compilerProps = ((IApplicationContext) context.getAppContext()).getCompilerProperties();
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.1.ddl.sqlpp
new file mode 100644
index 0000000..a80fc8a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.1.ddl.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type DataType as open {
+id:int,
+name:string,
+age:int,
+hobby:string
+};
+
+create type MetaType as closed {
+id:int
+};
+
+create dataset DS(DataType) with meta(MetaType) primary key meta().id;
+
+create feed DsStream with {
+ "adapter-name" : "localfs",
+ "reader" : "localfs",
+ "parser" : "record-with-metadata",
+ "type-name" : "DataType",
+ "meta-type-name" : "MetaType",
+ "path" : "asterix_nc1://data/csv/people.csv",
+ "format" : "csv",
+ "delimiter" : ",",
+ "record-format" : "adm",
+ "record-index" : "1",
+ "key-indexes" : "0",
+ "key-indicators" : "1",
+ "header" : "false"
+};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.2.update.sqlpp
new file mode 100644
index 0000000..6d41058
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+set `wait-for-completion-feed` "true";
+connect feed DsStream to dataset DS;
+start feed DsStream;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.3.query.sqlpp
new file mode 100644
index 0000000..b317302
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.3.query.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: use of meta() after group by requires meta() be assigned to a variable in a LET clause
+ * Result : success
+ */
+
+USE test;
+
+FROM DS AS d
+LET meta_id = meta().id
+GROUP BY d.hobby AS hobby
+SELECT hobby, COUNT(meta_id) AS count
+ORDER BY hobby;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp
new file mode 100644
index 0000000..bb016f7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: use of meta() after group by requires meta() be assigned to a variable in a LET clause
+ * Result : failure
+ */
+
+USE test;
+
+FROM DS AS d
+GROUP BY d.hobby AS hobby
+SELECT hobby, COUNT(meta().id)
+ORDER BY hobby;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.5.ddl.sqlpp
new file mode 100644
index 0000000..f12a2b7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.5.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/meta/meta_after_gby/meta_after_gby.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/meta/meta_after_gby/meta_after_gby.3.adm
new file mode 100644
index 0000000..bf1932c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/meta/meta_after_gby/meta_after_gby.3.adm
@@ -0,0 +1,6 @@
+{ "hobby": "basketball", "count": 1 }
+{ "hobby": "bowling", "count": 1 }
+{ "hobby": "hiking", "count": 2 }
+{ "hobby": "reading", "count": 2 }
+{ "hobby": "soccer", "count": 1 }
+{ "hobby": "tennis", "count": 2 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 17f4aee..ccd8f5a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -11358,6 +11358,12 @@
<output-dir compare="Text">resolving_pk_with_meta</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="meta_after_gby">
+ <output-dir compare="Text">meta_after_gby</output-dir>
+ <expected-error>Compilation error: Inappropriate use of function 'meta'. For example, after GROUP BY (in line 29, at column 21)</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="big-object">
<test-case FilePath="big-object">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
index b044293..06df020 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
@@ -192,7 +192,7 @@
IScalarEvaluatorFactory[] castEvalFact =
new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) };
IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1];
- sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact);
+ sefs[0] = createCastFunction(strictCast, sourceLoc).createEvaluatorFactory(castEvalFact);
AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
castAssign.setSourceLocation(sourceLoc);
return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign },
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 318814c..6bcc039 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -327,16 +327,16 @@
IScalarEvaluatorFactory[] castEvalFact =
new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) };
IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1];
- sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact);
+ sefs[0] = createCastFunction(strictCast, sourceLoc).createEvaluatorFactory(castEvalFact);
AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
castAssign.setSourceLocation(sourceLoc);
return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign },
new RecordDescriptor[] { enforcedRecDesc });
}
- protected IFunctionDescriptor createCastFunction(boolean strictCast) throws AlgebricksException {
+ IFunctionDescriptor createCastFunction(boolean strictCast, SourceLocation sourceLoc) throws AlgebricksException {
IFunctionDescriptor castFuncDesc = metadataProvider.getFunctionManager()
- .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
+ .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX, sourceLoc);
castFuncDesc.setSourceLocation(sourceLoc);
castFuncDesc.setImmutableStates(enforcedItemType, itemType);
return castFuncDesc;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
index e1657ff..0d0b3c6 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
@@ -20,6 +20,7 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.SourceLocation;
/**
* A registry of {@link IFunctionDescriptor} and {@link IFunctionTypeInferer}
@@ -27,7 +28,8 @@
*/
public interface IFunctionManager {
- IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException;
+ IFunctionDescriptor lookupFunction(FunctionIdentifier fid, SourceLocation sourceLocation)
+ throws AlgebricksException;
IFunctionTypeInferer lookupFunctionTypeInferer(FunctionIdentifier fid);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
index 8b25e06..0db1821 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
@@ -141,7 +141,8 @@
}
IScalarEvaluatorFactory fldIndexEvalFactory =
new ConstantEvalFactory(Arrays.copyOf(abvs.getByteArray(), abvs.getLength()));
- IFunctionDescriptor fDesc = functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_BY_INDEX);
+ IFunctionDescriptor fDesc =
+ functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_BY_INDEX, sourceLoc);
fDesc.setSourceLocation(sourceLoc);
fDesc.setImmutableStates(recType);
return fDesc.createEvaluatorFactory(
@@ -158,7 +159,8 @@
}
IScalarEvaluatorFactory fldNameEvalFactory =
new ConstantEvalFactory(Arrays.copyOf(abvs.getByteArray(), abvs.getLength()));
- IFunctionDescriptor fDesc = functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_BY_NAME);
+ IFunctionDescriptor fDesc =
+ functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_BY_NAME, sourceLoc);
fDesc.setSourceLocation(sourceLoc);
return fDesc.createEvaluatorFactory(
new IScalarEvaluatorFactory[] { recordEvalFactory, fldNameEvalFactory });
@@ -166,7 +168,7 @@
}
if (fldName.size() > 1) {
- IFunctionDescriptor fDesc = functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_NESTED);
+ IFunctionDescriptor fDesc = functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_NESTED, sourceLoc);
fDesc.setSourceLocation(sourceLoc);
fDesc.setImmutableStates(recType, fldName);
return fDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { recordEvalFactory });
@@ -241,7 +243,8 @@
}
IScalarEvaluatorFactory fldIndexEvalFactory =
new ConstantEvalFactory(Arrays.copyOf(abvs.getByteArray(), abvs.getLength()));
- IFunctionDescriptor fDesc = functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_BY_INDEX);
+ IFunctionDescriptor fDesc =
+ functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_BY_INDEX, sourceLoc);
fDesc.setSourceLocation(sourceLoc);
fDesc.setImmutableStates(recType);
IScalarEvaluatorFactory evalFactory = fDesc.createEvaluatorFactory(
@@ -267,7 +270,7 @@
} catch (HyracksDataException e) {
throw new AlgebricksException(e);
}
- IFunctionDescriptor fDesc = functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_NESTED);
+ IFunctionDescriptor fDesc = functionManager.lookupFunction(BuiltinFunctions.FIELD_ACCESS_NESTED, sourceLoc);
fDesc.setSourceLocation(sourceLoc);
fDesc.setImmutableStates(recType, fldName);
IScalarEvaluatorFactory evalFactory =
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionManager.java
index d5a6559..7158558 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionManager.java
@@ -22,6 +22,9 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.functions.IFunctionManager;
@@ -29,6 +32,7 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.SourceLocation;
/**
* Default implementation of {@link IFunctionManager}.
@@ -57,11 +61,15 @@
}
@Override
- public IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException {
+ public IFunctionDescriptor lookupFunction(FunctionIdentifier fid, SourceLocation src) throws AlgebricksException {
Pair<FunctionIdentifier, Integer> key = new Pair<>(fid, fid.getArity());
IFunctionDescriptorFactory factory = functions.get(key);
if (factory == null) {
- throw new AlgebricksException("Inappropriate use of function " + "'" + fid.getName() + "'");
+ String msg = "Inappropriate use of function '" + fid.getName() + "'";
+ if (fid.equals(BuiltinFunctions.META)) {
+ msg = msg + ". For example, after GROUP BY";
+ }
+ throw AsterixException.create(ErrorCode.COMPILATION_ERROR, src, msg);
}
return factory.createFunctionDescriptor();
}