[ASTERIXDB-3243][EXT]: Issue warning and skip file on computed field type mismatch
Change-Id: I2b21bfb22a59163a5e66201b45aaf3ad7ae7dda9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17719
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index e50ec3c..cd40b1a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -221,6 +221,7 @@
</compilation-unit>
</test-case>
<!-- Parquet Tests End -->
+ <!-- Dynamic prefixes tests start -->
<test-case FilePath="external-dataset/s3/filter">
<compilation-unit name="one-field">
<output-dir compare="Text">one-field</output-dir>
@@ -229,14 +230,15 @@
<test-case FilePath="external-dataset/s3/filter" check-warnings="true">
<compilation-unit name="type-mismatch">
<output-dir compare="Text">type-mismatch</output-dir>
- <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'java.lang.NumberFormatException: For input string: "department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'java.lang.NumberFormatException: For input string: "department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'java.lang.NumberFormatException: For input string: "department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File: 'external-filter/last-name/Jones/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File: 'external-filter/last-name/miller/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File: 'external-filter/last-name/smith/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For input string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For input string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For input string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/last-name/Jones/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For input string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/last-name/miller/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For input string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/last-name/smith/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For input string: "last-name"'</expected-warn>
</compilation-unit>
</test-case>
+ <!-- Dynamic prefixes tests end -->
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
index a07e067..40b10d4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
@@ -62,20 +62,16 @@
DataOutput output = value.getDataOutput();
SerializerDeserializerUtil.serializeTag(typeTag, output);
- try {
- switch (typeTag) {
- case TINYINT:
- case SMALLINT:
- case INTEGER:
- case BIGINT:
- Integer64SerializerDeserializer.write(Long.parseLong(stringValue), output);
- case DOUBLE:
- DoubleSerializerDeserializer.write(Double.parseDouble(stringValue), output);
- case STRING:
- stringSerDer.serialize(stringValue, output);
- }
- } catch (Exception ex) {
- throw HyracksDataException.create(ex);
+ switch (typeTag) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+ Integer64SerializerDeserializer.write(Long.parseLong(stringValue), output);
+ case DOUBLE:
+ DoubleSerializerDeserializer.write(Double.parseDouble(stringValue), output);
+ case STRING:
+ stringSerDer.serialize(stringValue, output);
}
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 66fa445..9d45fed 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -277,7 +277,7 @@
evaluator.setValue(i, computedFieldValue);
}
}
- } catch (HyracksDataException ex) {
+ } catch (NumberFormatException ex) {
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(null, ErrorCode.FAILED_TO_EVALUATE_COMPUTED_FIELD,
LogRedactionUtil.userData(key), computedFieldName, computedFieldType,