[ASTERIXDB-3392] Fix format in tests
Change-Id: I480866ba09e519f9d1242fdfdc9e2a70f7578f13
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18778
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Tested-by: Wail Alkowaileet <wael.y.k@gmail.com>
Integration-Tests: Wail Alkowaileet <wael.y.k@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
index 5fb0a9b..ce68cbd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
@@ -27,15 +27,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks3")
-TYPE ( {
- id : string,
- name : string,
- nested :
- {
- first : string,
- second : string
- }
- }
+TYPE ( {id : string, name : string, nested: { first : string, second : string} }
)
WITH {
"accessKeyId":"dummyAccessKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
index 8b1efc8..5e8d730 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
@@ -26,14 +26,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks4")
-TYPE ( {
- id : bigint,
- name : string,
- nested:
- {
- first : string
- }
- } )
+TYPE ( {id : bigint,name : string, nested: { first : string } } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
index 79b2d1a..cda2519 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
@@ -25,11 +25,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks5")
-TYPE ( {
- id : bigint,
- name : string,
- nested : string
- } )
+TYPE ( { id : bigint, name : string, nested : string } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
index 3e6ac48..a6db11d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
@@ -25,16 +25,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks6")
-TYPE ( {
- id : bigint,
- name : {
- first : string
- },
- nested:{
- first : string,
- second : string
- }
- } )
+TYPE ( {id : bigint,name : { first : string },nested:{first : string,second : string} } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
index f22071a..ca2cfc1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
@@ -24,9 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks11")
-TYPE ( {
- id : int , name : binary
- } )
+TYPE ( { id : int , name : binary } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
index d5d11eb..86e55e5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
@@ -24,10 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks12")
-TYPE ( {
- id : int,
- name : string
- } )
+TYPE ( { id : int, name : string } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
index 75245f1..281610b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
@@ -24,11 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks13")
-TYPE ( {
- id : int,
- name : string,
- list : [int,string]
- } )
+TYPE ( { id : int, name : string, list : [int,string]} )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
index 0becb36..1d1c6af 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
@@ -24,11 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks14")
-TYPE ( {
- id : int,
- name : string,
- list : [int |
- } )
+TYPE ( {id : int, name : string, list : [int |} )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
index cc67f79..a26624a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
@@ -24,11 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks15")
-TYPE ( {
- id : int,
- name : string,
- list : [int] )
-
+TYPE ( {id : int, name : string, list : [int] )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 1107dda..d02d551 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -117,7 +117,7 @@
<expected-error>ASX1202: Unsupported compression scheme rar. Supported schemes for parquet are [gzip, snappy, zstd]</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
<expected-error>ASX1204: 'binary' type not supported in parquet format</expected-error>
- <expected-error>ASX1205: Invalid Parquet Writer Version provided.Supported values: 1,2</expected-error>
+ <expected-error>ASX1205: Invalid Parquet Writer Version provided '3'. Supported values: [1, 2]</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 716dcf6..748a2b9 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -309,7 +309,7 @@
1202 = Unsupported compression scheme %1$s. Supported schemes for %2$s are %3$s
1203 = Invalid schema provided: '%1$s'
1204 = '%1$s' type not supported in parquet format
-1205 = Invalid Parquet Writer Version provided.Supported values: 1,2
+1205 = Invalid Parquet Writer Version provided '%1$s'. Supported values: %2$s
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 02c2070..0407005 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -336,6 +336,7 @@
public static final Set<String> WRITER_SUPPORTED_ADAPTERS;
public static final Set<String> TEXTUAL_WRITER_SUPPORTED_COMPRESSION;
public static final Set<String> PARQUET_WRITER_SUPPORTED_COMPRESSION;
+ public static final Set<String> PARQUET_WRITER_SUPPORTED_VERSION;
public static final int PARQUET_DICTIONARY_PAGE_SIZE = 1048576;
static {
@@ -345,6 +346,7 @@
TEXTUAL_WRITER_SUPPORTED_COMPRESSION = Set.of(KEY_COMPRESSION_GZIP);
PARQUET_WRITER_SUPPORTED_COMPRESSION =
Set.of(KEY_COMPRESSION_GZIP, KEY_COMPRESSION_SNAPPY, KEY_COMPRESSION_ZSTD);
+ PARQUET_WRITER_SUPPORTED_VERSION = Set.of(PARQUET_WRITER_VERSION_VALUE_1, PARQUET_WRITER_VERSION_VALUE_2);
}
public static class ParquetOptions {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
index 5c3585b..5059ec8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
@@ -27,8 +27,6 @@
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_PARQUET_ROW_GROUP_SIZE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_WRITER_MAX_RESULT;
import static org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_KEY;
-import static org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_1;
-import static org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2;
import static org.apache.asterix.external.util.ExternalDataConstants.WRITER_MAX_RESULT_MINIMUM;
import java.util.List;
@@ -79,18 +77,14 @@
validateParquetCompression(configuration, sourceLocation);
validateParquetRowGroupSize(configuration);
validateParquetPageSize(configuration);
- validateVersion(configuration);
+ validateVersion(configuration, sourceLocation);
}
- private static void validateVersion(Map<String, String> configuration) throws CompilationException {
+ private static void validateVersion(Map<String, String> configuration, SourceLocation sourceLocation)
+ throws CompilationException {
String version = configuration.get(PARQUET_WRITER_VERSION_KEY);
- if (version == null) {
- return;
- }
- if (version.equals(PARQUET_WRITER_VERSION_VALUE_1) || version.equals(PARQUET_WRITER_VERSION_VALUE_2)) {
- return;
- }
- throw CompilationException.create(ErrorCode.INVALID_PARQUET_WRITER_VERSION);
+ checkSupported(PARQUET_WRITER_VERSION_KEY, version, ExternalDataConstants.PARQUET_WRITER_SUPPORTED_VERSION,
+ ErrorCode.INVALID_PARQUET_WRITER_VERSION, sourceLocation, true);
}
private static void validateParquetRowGroupSize(Map<String, String> configuration) throws CompilationException {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
index 410c951..0dcdb3a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
@@ -31,7 +31,7 @@
Map.entry(ATypeTag.STRING, PrimitiveType.PrimitiveTypeName.BINARY),
Map.entry(ATypeTag.TINYINT, PrimitiveType.PrimitiveTypeName.INT32),
Map.entry(ATypeTag.SMALLINT, PrimitiveType.PrimitiveTypeName.INT32),
- Map.entry(ATypeTag.INTEGER, PrimitiveType.PrimitiveTypeName.INT64),
+ Map.entry(ATypeTag.INTEGER, PrimitiveType.PrimitiveTypeName.INT32),
Map.entry(ATypeTag.BIGINT, PrimitiveType.PrimitiveTypeName.INT64),
Map.entry(ATypeTag.FLOAT, PrimitiveType.PrimitiveTypeName.FLOAT),
Map.entry(ATypeTag.DOUBLE, PrimitiveType.PrimitiveTypeName.DOUBLE),
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
index 23b9f93..ee7b3fc 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
@@ -157,14 +157,12 @@
}
private static ParquetProperties.WriterVersion getParquetWriterVersion(Map<String, String> configuration) {
-
- if (configuration.get(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY) == null) {
- return ParquetProperties.WriterVersion.PARQUET_1_0;
- } else if (configuration.get(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY)
- .equals(ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2)) {
+ String writerVersionString = configuration.getOrDefault(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY,
+ ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_1);
+ if (writerVersionString.equals(ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2)) {
return ParquetProperties.WriterVersion.PARQUET_2_0;
- } else
- return ParquetProperties.WriterVersion.PARQUET_1_0;
+ }
+ return ParquetProperties.WriterVersion.PARQUET_1_0;
}
private static String getRowGroupSize(Map<String, String> configuration) {