[ASTERIXDB-2735][EXT] Support boolean value for properties of external data
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
The configuration properties of the external data in the DDL statements are
supplied as key-value pairs of strings. The user should be able to supply
normal boolean values for boolean properties such as "header" which tells
whether the data source has a header or not (for delimited data such as CSV).
Change-Id: I0470f7f0bbf7476e1fbd6b9f124355cf8b00c17c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6343
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp
index 54022fc..cef7274 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp
@@ -33,5 +33,5 @@
("container"="playground"),
("definition"="data_dir"),
("format"="CSV"),
-("header"="true")
+("header"=true)
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp
index 047a540..6f96ecf 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp
@@ -29,5 +29,5 @@
("definition"="data_dir"),
("format"="CSV"),
("header"="false"),
-("redact-warnings"="true")
+("redact-warnings"="True")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp
index 575d46f..47daa38 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp
@@ -29,6 +29,6 @@
("definition"="data_dir"),
("format"="CSV"),
("header"="true"),
-("redact-warnings"="true"),
+("redact-warnings"=True),
("null"="\\N")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp
index 54e0dc4..a63f3bc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp
@@ -33,6 +33,6 @@
("container"="playground"),
("definition"="csv-data/reviews/csv"),
("format"="Csv"),
-("header"="false"),
+("header"=False),
("null"="")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp
index 413987d..b8df075 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp
@@ -33,6 +33,6 @@
("container"="playground"),
("definition"="csv-data/reviews/gz"),
("format"="Csv"),
-("header"="false"),
+("header"=false),
("null"="")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8fb17bc..2ae2838 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -23,7 +23,6 @@
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START;
-import static org.apache.asterix.external.util.ExternalDataConstants.KEY_REDACT_WARNINGS;
import java.util.EnumMap;
import java.util.Map;
@@ -370,11 +369,10 @@
configuration.put(ExternalDataConstants.KEY_FORMAT, lowerCaseFormat);
}
}
- // normalize the "header" parameter
- paramValue = configuration.get(ExternalDataConstants.KEY_HEADER);
- if (paramValue != null) {
- configuration.put(ExternalDataConstants.KEY_HEADER, paramValue.toLowerCase().trim());
- }
+ // normalize "header" parameter
+ putToLowerIfExists(configuration, ExternalDataConstants.KEY_HEADER);
+ // normalize "redact-warnings" parameter
+ putToLowerIfExists(configuration, ExternalDataConstants.KEY_REDACT_WARNINGS);
}
/**
@@ -395,9 +393,10 @@
char delimiter = validateGetDelimiter(configuration);
validateGetQuote(configuration, delimiter);
validateGetEscape(configuration);
- String value = configuration.get(KEY_REDACT_WARNINGS);
+ String value = configuration.get(ExternalDataConstants.KEY_REDACT_WARNINGS);
if (value != null && !isBoolean(value)) {
- throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, KEY_REDACT_WARNINGS, value);
+ throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, ExternalDataConstants.KEY_REDACT_WARNINGS,
+ value);
}
}
@@ -430,4 +429,11 @@
throw new RuntimeDataException(ErrorCode.INVALID_CHAR_LENGTH, parameterValue, parameterName);
}
}
+
+ private static void putToLowerIfExists(Map<String, String> configuration, String key) {
+ String paramValue = configuration.get(key);
+ if (paramValue != null) {
+ configuration.put(key, paramValue.toLowerCase().trim());
+ }
+ }
}
diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
index 127dc62..4bf4ea4 100644
--- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -1238,7 +1238,9 @@
String value;
}
{
- <LEFTPAREN> key = StringLiteral() <EQ> value = StringLiteral() <RIGHTPAREN>
+ <LEFTPAREN> key = StringLiteral()
+ <EQ>( value = StringLiteral() | (<TRUE> | <FALSE>) {value = token.image.toLowerCase();} )
+ <RIGHTPAREN>
{
return new Pair<String, String>(key, value);
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 107e240..7538887 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -1564,7 +1564,9 @@
String value;
}
{
- <LEFTPAREN> key = ConstantString() <EQ> value = ConstantString() <RIGHTPAREN>
+ <LEFTPAREN> key = ConstantString()
+ <EQ> ( value = ConstantString() | (<TRUE> | <FALSE>) {value = token.image.toLowerCase();} )
+ <RIGHTPAREN>
{
return new Pair<String, String>(key, value);
}