Merge branch 'cheshire-cat'
Change-Id: I72241c264b37ead5ae1fc40ed1f2b50f20c8689c
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
index 4d726cf..bad3f79 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
@@ -64,6 +64,6 @@
@Override
public String toString() {
- return getClass().getSimpleName();
+ return getClass().getSimpleName() + "{" + "kind=" + kind + ", runtimeId=" + runtimeId + '}';
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
index 1e5d805..54a1bb7 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
@@ -190,13 +190,11 @@
if (!assignVars.isEmpty()) {
usedVarList.clear();
expr.getUsedVariables(usedVarList);
- for (int i = 0, ln = assignVars.size(); i < ln; i++) {
- List<LogicalVariable> candidateVarList = assignVars.get(i);
- if (OperatorPropertiesUtil.disjoint(candidateVarList, usedVarList)) {
- assignVarList = candidateVarList;
- assignExprList = assignExprs.get(i);
- break;
- }
+ int candidateVarListIdx = assignVars.size() - 1;
+ List<LogicalVariable> candidateVarList = assignVars.get(candidateVarListIdx);
+ if (OperatorPropertiesUtil.disjoint(candidateVarList, usedVarList)) {
+ assignVarList = candidateVarList;
+ assignExprList = assignExprs.get(candidateVarListIdx);
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
index d3eb0c2..22d87ac 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
@@ -22,6 +22,7 @@
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.ExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.base.TypeCastUtils;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
@@ -82,6 +83,9 @@
if (reqArgType.getTypeTag() == ATypeTag.OBJECT) {
castFlag = !IntroduceDynamicTypeCastRule.compatible((ARecordType) reqArgType, inputType,
argExpr.getValue().getSourceLocation());
+ } else if (reqArgType.getTypeTag() == ATypeTag.ANY) {
+ IAType inputPrimeType = TypeComputeUtils.getActualType(inputType);
+ castFlag = inputPrimeType.getTypeTag().isDerivedType();
} else {
castFlag = !reqArgType.equals(inputType);
}
@@ -115,6 +119,13 @@
if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
return false;
}
- return op.acceptExpressionTransform(expr -> rewriteFunctionArgs(op, expr, context));
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ boolean applied = op.acceptExpressionTransform(expr -> rewriteFunctionArgs(op, expr, context));
+ if (applied) {
+ context.addToDontApplySet(this, op);
+ }
+ return applied;
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
index 4ad1040..5c743ee 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
@@ -73,4 +73,6 @@
* inconsistent.
*/
boolean isForceDropDataset();
+
+ boolean isSkipAdmissionPolicy();
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index 083fa83..d60b791 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -273,6 +273,9 @@
* @param statementParameters
* Statement parameters
* @param statementRewriter
+ * The statement rewriter
+ * @param requestParameters
+ * The request parameters
* @return the compiled {@code JobSpecification}
* @throws AsterixException
* @throws RemoteException
@@ -281,7 +284,8 @@
*/
JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider,
Query query, ICompiledDmlStatement dmlStatement, Map<String, IAObject> statementParameters,
- IStatementRewriter statementRewriter) throws RemoteException, AlgebricksException, ACIDException;
+ IStatementRewriter statementRewriter, IRequestParameters requestParameters)
+ throws RemoteException, AlgebricksException, ACIDException;
/**
* returns the active dataverse for an entity or a statement
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 0141cc5..326ddcc 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -77,6 +77,7 @@
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.ExecutionPlans;
+import org.apache.asterix.translator.IRequestParameters;
import org.apache.asterix.translator.SessionConfig;
import org.apache.asterix.translator.SessionOutput;
import org.apache.asterix.translator.SqlppExpressionToPlanTranslator;
@@ -205,7 +206,8 @@
public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider,
Query query, int varCounter, String outputDatasetName, SessionOutput output,
ICompiledDmlStatement statement, Map<VarIdentifier, IAObject> externalVars, IResponsePrinter printer,
- IWarningCollector warningCollector) throws AlgebricksException, ACIDException {
+ IWarningCollector warningCollector, IRequestParameters requestParameters)
+ throws AlgebricksException, ACIDException {
// establish facts
final boolean isQuery = query != null;
@@ -307,15 +309,17 @@
JobSpecification spec = compiler.createJob(metadataProvider.getApplicationContext(), jobEventListenerFactory);
if (isQuery) {
- // Sets a required capacity, only for read-only queries.
- // DDLs and DMLs are considered not that frequent.
- // limit the computation locations to the locations that will be used in the query
- final INodeJobTracker nodeJobTracker = metadataProvider.getApplicationContext().getNodeJobTracker();
- final AlgebricksAbsolutePartitionConstraint jobLocations =
- getJobLocations(spec, nodeJobTracker, computationLocations);
- final IClusterCapacity jobRequiredCapacity =
- ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
- spec.setRequiredClusterCapacity(jobRequiredCapacity);
+ if (requestParameters == null || !requestParameters.isSkipAdmissionPolicy()) {
+ // Sets a required capacity, only for read-only queries.
+ // DDLs and DMLs are considered not that frequent.
+ // limit the computation locations to the locations that will be used in the query
+ final INodeJobTracker nodeJobTracker = metadataProvider.getApplicationContext().getNodeJobTracker();
+ final AlgebricksAbsolutePartitionConstraint jobLocations =
+ getJobLocations(spec, nodeJobTracker, computationLocations);
+ final IClusterCapacity jobRequiredCapacity =
+ ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
+ spec.setRequiredClusterCapacity(jobRequiredCapacity);
+ }
}
if (isQuery && conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
generateJob(spec);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
index cf287d9..5a3cde6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
@@ -134,7 +134,7 @@
}
URI createDownloadURI(Path file) throws Exception {
- String path = paths[0].substring(0, trims[0]) + GET_UDF_DIST_ENDPOINT + '/' + file.getFileName();
+ String path = paths[0].substring(0, servletPathLengths[0]) + GET_UDF_DIST_ENDPOINT + '/' + file.getFileName();
String host = getHyracksClientConnection().getHost();
return new URI(httpServerProtocol.toString(), null, host, httpServerPort, path, null, null);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 2891710..e18e50c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -133,15 +133,12 @@
this.locations = locations;
this.numRegistered = 0;
this.numDeRegistered = 0;
- this.handler =
- (ActiveNotificationHandler) metadataProvider.getApplicationContext().getActiveNotificationHandler();
+ this.handler = (ActiveNotificationHandler) appCtx.getActiveNotificationHandler();
handler.registerListener(this);
}
protected synchronized void setState(ActivityState newState) {
- if (LOGGER.isEnabled(level)) {
- LOGGER.log(level, "State of " + getEntityId() + "is being set to " + newState + " from " + state);
- }
+ LOGGER.log(level, "State of {} is being set to {} from {}", getEntityId(), newState, state);
this.prevState = state;
this.state = newState;
if (newState == ActivityState.STARTING || newState == ActivityState.RECOVERING
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 6318f8e..3351075 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -36,19 +37,26 @@
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
- RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories) throws AlgebricksException {
+ RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
+ AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
+ this.storageLocations = storageLocations;
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return storageLocations;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new DumpIndexFunction(metadataProvider.getClusterLocations(), indexDataflowHelperFactory, recDesc,
- comparatorFactories);
+ return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index 000e910..30eaf93 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -26,6 +26,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.SecondaryIndexOperationsHelper;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -67,7 +68,10 @@
IndexDataflowHelperFactory indexDataflowHelperFactory =
new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
secondaryIndexHelper.getSecondaryFileSplitProvider());
+ AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
+ (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
- secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories());
+ secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
+ secondaryPartitionConstraint);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index d3345fb..29ee76d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -87,12 +87,24 @@
private final ProfileType profileType;
private final IRequestReference requestReference;
private final boolean forceDropDataset;
+ private final boolean skipAdmissionPolicy;
public ExecuteStatementRequestMessage(String requestNodeId, long requestMessageId, ILangExtension.Language lang,
String statementsText, SessionConfig sessionConfig, ResultProperties resultProperties,
String clientContextID, String handleUrl, Map<String, String> optionalParameters,
Map<String, byte[]> statementParameters, boolean multiStatement, ProfileType profileType,
int statementCategoryRestrictionMask, IRequestReference requestReference, boolean forceDropDataset) {
+ this(requestNodeId, requestMessageId, lang, statementsText, sessionConfig, resultProperties, clientContextID,
+ handleUrl, optionalParameters, statementParameters, multiStatement, profileType,
+ statementCategoryRestrictionMask, requestReference, forceDropDataset, false);
+ }
+
+ protected ExecuteStatementRequestMessage(String requestNodeId, long requestMessageId, ILangExtension.Language lang,
+ String statementsText, SessionConfig sessionConfig, ResultProperties resultProperties,
+ String clientContextID, String handleUrl, Map<String, String> optionalParameters,
+ Map<String, byte[]> statementParameters, boolean multiStatement, ProfileType profileType,
+ int statementCategoryRestrictionMask, IRequestReference requestReference, boolean forceDropDataset,
+ boolean skipAdmissionPolicy) {
this.requestNodeId = requestNodeId;
this.requestMessageId = requestMessageId;
this.lang = lang;
@@ -108,6 +120,7 @@
this.profileType = profileType;
this.requestReference = requestReference;
this.forceDropDataset = forceDropDataset;
+ this.skipAdmissionPolicy = skipAdmissionPolicy;
}
@Override
@@ -150,9 +163,10 @@
final IStatementExecutor.StatementProperties statementProperties =
new IStatementExecutor.StatementProperties();
Map<String, IAObject> stmtParams = RequestParameters.deserializeParameterValues(statementParameters);
- final IRequestParameters requestParameters = new RequestParameters(requestReference, statementsText, null,
- resultProperties, stats, statementProperties, outMetadata, clientContextID, optionalParameters,
- stmtParams, multiStatement, statementCategoryRestrictionMask, forceDropDataset);
+ final IRequestParameters requestParameters =
+ new RequestParameters(requestReference, statementsText, null, resultProperties, stats,
+ statementProperties, outMetadata, clientContextID, optionalParameters, stmtParams,
+ multiStatement, statementCategoryRestrictionMask, forceDropDataset, skipAdmissionPolicy);
translator.compileAndExecute(ccApp.getHcc(), requestParameters);
translator.getWarnings(warnings, maxWarnings - warnings.size());
stats.updateTotalWarningsCount(parserTotalWarningsCount);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 65cb36a..0359cf1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -173,15 +173,14 @@
deleteRecoveryTemporaryFiles();
//get active partitions on this node
- replayPartitionsLogs(partitions, logMgr.getLogReader(true), lowWaterMarkLSN);
+ replayPartitionsLogs(partitions, logMgr.getLogReader(true), lowWaterMarkLSN, true);
}
- @Override
- public synchronized void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN)
- throws IOException, ACIDException {
+ public synchronized void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN,
+ boolean closeOnFlushRedo) throws IOException, ACIDException {
try {
Set<Long> winnerJobSet = startRecoverysAnalysisPhase(partitions, logReader, lowWaterMarkLSN);
- startRecoveryRedoPhase(partitions, logReader, lowWaterMarkLSN, winnerJobSet);
+ startRecoveryRedoPhase(partitions, logReader, lowWaterMarkLSN, winnerJobSet, closeOnFlushRedo);
} finally {
logReader.close();
deleteRecoveryTemporaryFiles();
@@ -277,7 +276,7 @@
}
private synchronized void startRecoveryRedoPhase(Set<Integer> partitions, ILogReader logReader,
- long lowWaterMarkLSN, Set<Long> winnerTxnSet) throws IOException, ACIDException {
+ long lowWaterMarkLSN, Set<Long> winnerTxnSet, boolean closeOnFlushRedo) throws IOException, ACIDException {
int redoCount = 0;
long txnId = 0;
@@ -299,6 +298,7 @@
TxnEntityId tempKeyTxnEntityId = new TxnEntityId(-1, -1, -1, null, -1, false);
ILogRecord logRecord = null;
+ Set<Integer> flushRedoDatasets = new HashSet<>();
try {
logReader.setPosition(lowWaterMarkLSN);
logRecord = logReader.next();
@@ -409,6 +409,7 @@
&& !index.isCurrentMutableComponentEmpty()) {
// schedule flush
redoFlush(index, logRecord);
+ flushRedoDatasets.add(datasetId);
redoCount++;
} else {
// TODO: update checkpoint file?
@@ -441,6 +442,11 @@
for (long r : resourceIdList) {
datasetLifecycleManager.close(resourcesMap.get(r).getPath());
}
+ if (closeOnFlushRedo) {
+ // close datasets of indexes to ensure any cached state that might've been changed by recovery is cleared
+ // e.g. when redoing a flush, the component id generator needs to be reinitialized
+ datasetLifecycleManager.closeDatasets(flushRedoDatasets);
+ }
}
}
@@ -525,7 +531,7 @@
if (minLSN < readableSmallestLSN) {
minLSN = readableSmallestLSN;
}
- replayPartitionsLogs(partitions, logMgr.getLogReader(true), minLSN);
+ replayPartitionsLogs(partitions, logMgr.getLogReader(true), minLSN, false);
if (flush) {
appCtx.getDatasetLifecycleManager().flushAllDatasets();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java
index 18d303c..715e626 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java
@@ -33,12 +33,14 @@
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.client.utils.URIUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.control.CcId;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.service.IControllerService;
+import org.apache.hyracks.util.NetworkUtil;
import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -51,7 +53,7 @@
public RetrieveLibrariesTask(List<Pair<URI, String>> nodes) {
this.nodes = nodes;
- if (nodes.size() <= 0) {
+ if (nodes.isEmpty()) {
throw new IllegalArgumentException("No nodes specified to retrieve from");
}
}
@@ -62,7 +64,8 @@
boolean success = false;
for (Pair<URI, String> referenceNode : nodes) {
try {
- LOGGER.info("Retrieving UDFs from " + referenceNode.getFirst().getHost());
+ LOGGER.info("Retrieving UDFs from "
+ + NetworkUtil.toHostPort(URIUtils.extractHost(referenceNode.getFirst())));
retrieveLibrary(referenceNode.getFirst(), referenceNode.getSecond(), appContext);
success = true;
break;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 8c4675c..8f6b13f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -3186,7 +3186,7 @@
loadStmt.getDatasetName(), loadStmt.getAdapter(), properties, loadStmt.dataIsAlreadySorted());
cls.setSourceLocation(stmt.getSourceLocation());
JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls,
- null, responsePrinter, warningCollector);
+ null, responsePrinter, warningCollector, null);
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
@@ -3285,8 +3285,8 @@
CompiledDeleteStatement clfrqs = new CompiledDeleteStatement(stmtDelete.getVariableExpr(), dataverseName,
datasetName, stmtDelete.getCondition(), stmtDelete.getVarCounter(), stmtDelete.getQuery());
clfrqs.setSourceLocation(stmt.getSourceLocation());
- JobSpecification jobSpec =
- rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs, stmtParams, stmtRewriter);
+ JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs, stmtParams,
+ stmtRewriter, null);
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -3309,7 +3309,7 @@
@Override
public JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector,
MetadataProvider metadataProvider, Query query, ICompiledDmlStatement stmt,
- Map<String, IAObject> stmtParams, IStatementRewriter stmtRewriter)
+ Map<String, IAObject> stmtParams, IStatementRewriter stmtRewriter, IRequestParameters requestParameters)
throws AlgebricksException, ACIDException {
Map<VarIdentifier, IAObject> externalVars = createExternalVariables(stmtParams, stmtRewriter);
@@ -3321,7 +3321,7 @@
// Query Compilation (happens under the same ongoing metadata transaction)
return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, (Query) rewrittenResult.first,
rewrittenResult.second, stmt == null ? null : stmt.getDatasetName(), sessionOutput, stmt, externalVars,
- responsePrinter, warningCollector);
+ responsePrinter, warningCollector, requestParameters);
}
private JobSpecification rewriteCompileInsertUpsert(IClusterInfoCollector clusterInfoCollector,
@@ -3360,7 +3360,7 @@
// transaction)
return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, rewrittenInsertUpsert.getQuery(),
rewrittenResult.second, datasetName, sessionOutput, clfrqs, externalVars, responsePrinter,
- warningCollector);
+ warningCollector, null);
}
protected void handleCreateFeedStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception {
@@ -3824,8 +3824,8 @@
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- final JobSpecification jobSpec =
- rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams, stmtRewriter);
+ final JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams,
+ stmtRewriter, requestParameters);
// update stats with count of compile-time warnings. needs to be adapted for multi-statement.
stats.updateTotalWarningsCount(warningCollector.getTotalWarningsCount());
afterCompile();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index d5ea685..6c8f21c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -59,6 +59,7 @@
private final int statementCategoryRestrictionMask;
private final String statement;
private final boolean forceDropDataset;
+ private final boolean skipAdmissionPolicy;
public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
@@ -83,6 +84,16 @@
IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
int statementCategoryRestrictionMask, boolean forceDropDataset) {
+ this(requestReference, statement, resultSet, resultProperties, stats, statementProperties, outMetadata,
+ clientContextId, optionalParameters, statementParameters, multiStatement,
+ statementCategoryRestrictionMask, forceDropDataset, false);
+ }
+
+ public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
+ IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
+ Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
+ int statementCategoryRestrictionMask, boolean forceDropDataset, boolean skipAdmissionPolicy) {
this.requestReference = requestReference;
this.statement = statement;
this.resultSet = resultSet;
@@ -96,6 +107,7 @@
this.multiStatement = multiStatement;
this.statementCategoryRestrictionMask = statementCategoryRestrictionMask;
this.forceDropDataset = forceDropDataset;
+ this.skipAdmissionPolicy = skipAdmissionPolicy;
}
@Override
@@ -149,6 +161,11 @@
}
@Override
+ public boolean isSkipAdmissionPolicy() {
+ return skipAdmissionPolicy;
+ }
+
+ @Override
public Map<String, IAObject> getStatementParameters() {
return statementParameters;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
index dcd52a0..9472da5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
@@ -253,7 +253,7 @@
clfrqs = new CompiledStatements.CompiledUpsertStatement(feedConn.getDataverseName(),
feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null);
}
- return statementExecutor.rewriteCompileQuery(hcc, metadataProvider, feedConnQuery, clfrqs, null, null);
+ return statementExecutor.rewriteCompileQuery(hcc, metadataProvider, feedConnQuery, clfrqs, null, null, null);
}
private static JobSpecification combineIntakeCollectJobs(MetadataProvider metadataProvider, Feed feed,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index 2450025..e32f8ea 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -33,6 +33,7 @@
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.client.utils.URIUtils;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
@@ -78,7 +79,7 @@
private HttpClientContext createHttpClientContext(URI path, Pair<String, String> credentials) {
HttpClientContext hcCtx = HttpClientContext.create();
- HttpHost h = new HttpHost(path.getHost(), path.getPort(), "http");
+ HttpHost h = URIUtils.extractHost(path);
CredentialsProvider cp = new BasicCredentialsProvider();
cp.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(credentials.first, credentials.second));
hcCtx.setCredentialsProvider(cp);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 28b7fb3..40946d3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -106,7 +106,6 @@
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
@@ -119,6 +118,7 @@
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.client.utils.URIUtils;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
@@ -207,10 +207,7 @@
private static final int MAX_NON_UTF_8_STATEMENT_SIZE = 64 * 1024;
private static final ContentType TEXT_PLAIN_UTF8 = ContentType.create(HttpUtil.ContentType.APPLICATION_JSON, UTF_8);
- private final IPollTask plainExecutor = (testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath, expectedWarnings) -> executeTestFile(testCaseCtx,
- ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount, expectedResultFileCtxs,
- testFile, actualPath, expectedWarnings);
+ private final IPollTask plainExecutor = this::executeTestFile;
public static final String DELIVERY_ASYNC = "async";
public static final String DELIVERY_DEFERRED = "deferred";
@@ -675,7 +672,7 @@
cp.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(credentials.first, credentials.second));
HttpClientContext hcCtx = HttpClientContext.create();
AuthCache ac = new BasicAuthCache();
- ac.put(new HttpHost(method.getURI().getHost(), method.getURI().getPort(), "http"), new BasicScheme());
+ ac.put(URIUtils.extractHost(method.getURI()), new BasicScheme());
hcCtx.setAuthCache(ac);
CloseableHttpClient client = HttpClients.custom().setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE)
.setDefaultCredentialsProvider(cp).build();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index 9c6e95e..2aad416 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -23,7 +23,9 @@
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.Semaphore;
import org.apache.asterix.app.bootstrap.TestNodeController;
@@ -62,6 +64,7 @@
import org.apache.hyracks.storage.am.lsm.btree.impl.ITestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -143,8 +146,7 @@
checkComponentIds();
// insert more records
createInsertOps();
- insertRecords(PARTITION_0, StorageTestUtils.RECORDS_PER_COMPONENT, StorageTestUtils.RECORDS_PER_COMPONENT,
- true);
+ insertRecords(PARTITION_0, StorageTestUtils.TOTAL_NUM_OF_RECORDS, StorageTestUtils.RECORDS_PER_COMPONENT, true);
dsInfo.waitForIO();
checkComponentIds();
@@ -487,8 +489,14 @@
List<ILSMDiskComponent> secondaryDiskComponents = secondaryIndexes[partitionIndex].getDiskComponents();
Assert.assertEquals(primaryDiskComponents.size(), secondaryDiskComponents.size());
+ Set<ILSMComponentId> uniqueIds = new HashSet<>();
for (int i = 0; i < primaryDiskComponents.size(); i++) {
Assert.assertEquals(primaryDiskComponents.get(i).getId(), secondaryDiskComponents.get(i).getId());
+ ILSMComponentId id = primaryDiskComponents.get(i).getId();
+ boolean added = uniqueIds.add(id);
+ if (!added) {
+ throw new IllegalStateException("found duplicate component ids: " + id);
+ }
}
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
new file mode 100644
index 0000000..13cfe8a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("accessKeyId"="dummyAccessKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
new file mode 100644
index 0000000..b8d0945
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
new file mode 100644
index 0000000..9eda057
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
new file mode 100644
index 0000000..69a0b25
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
index a8ba8a1..8041b5e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
@@ -18,6 +18,8 @@
*/
use externallibtest;
-create function typeValidation(a, b, c, d, e, f, g)
+create function typeValidation(a, b, c, d, e, f, g, h)
as "roundtrip", "Tests.roundtrip" at testlib;
+create function typeValidationNullCall(a, b, c, d, e, f, g, h)
+ as "roundtrip", "Tests.roundtrip" at testlib with {"null-call": true};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
index 0ae7d0c..f9aaf57 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
new file mode 100644
index 0000000..884f31f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
new file mode 100644
index 0000000..fdd71ef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
new file mode 100644
index 0000000..b85e0d0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
new file mode 100644
index 0000000..fa76f1a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
new file mode 100644
index 0000000..30640fd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
new file mode 100644
index 0000000..810ba3c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.0.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.0.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.0.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.0.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.1.lib.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.1.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.1.lib.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.2.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.2.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.2.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.3.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.3.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.3.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.4.query.sqlpp
new file mode 100644
index 0000000..7ebfc7a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Testcase for ASTERIXDB-2884
+ */
+
+use externallibtest;
+
+sqrt(sqrt(null));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.5.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.4.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.5.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.0.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.0.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp
new file mode 100644
index 0000000..b2bf929
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp
new file mode 100644
index 0000000..c7d6768
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+create function typeName(v) returns string
+ as "org.apache.asterix.external.library.TypeNameFactory" at testlib;
+
+create function typeNameNullCall(v) returns string
+ as "org.apache.asterix.external.library.TypeNameFactory" at testlib with {"null-call": true};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp
new file mode 100644
index 0000000..2542d55
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+{
+ "missing": typeName(missing) is missing,
+ "missing_nullcall": typeNameNullCall(missing),
+ "null": typeName(null) is null,
+ "null_nullcall": typeNameNullCall(null),
+ "boolean": typeName(true),
+ "string": typeName("x")
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp
new file mode 100644
index 0000000..ab6b723
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test1 if exists;
+create dataverse test1;
+use test1;
+
+create dataset a(
+ c_a1 bigint not unknown,
+ c_a2 bigint,
+ c_a3 string
+) primary key c_a1;
+
+create index ia2 on a(c_a2);
+
+create index ia3 on a(c_a3);
+
+create dataset b(
+ c_b1 bigint not unknown,
+ c_b2 bigint,
+ c_b3 string
+) primary key c_b1;
+
+create index ib2 on b(c_b2);
+
+create index ib3 on b(c_b3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp
new file mode 100644
index 0000000..621d057
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test2 if exists;
+create dataverse test2;
+use test2;
+
+create dataset a(
+ c_a1 bigint not unknown,
+ c_a2 bigint,
+ c_a3 string
+) primary key c_a1;
+
+create index ia2 on a(c_a2);
+
+create index ia3 on a(c_a3);
+
+create dataset b(
+ c_b1 bigint not unknown,
+ c_b2 bigint,
+ c_b3 string
+) primary key c_b1;
+
+create index ib2 on b(c_b2);
+
+create index ib3 on b(c_b3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp
new file mode 100644
index 0000000..e48a1e1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select ds.DataverseName, ds.DatasetName, dt.Derived.IsAnonymous, indexes
+from Metadata.`Dataset` as ds left join Metadata.`Datatype` dt
+on ds.DataverseName = dt.DataverseName and ds.DatatypeName = dt.DatatypeName
+let indexes = (
+ select idx.IndexName
+ from Metadata.`Index` as idx
+ where idx.DataverseName = ds.DataverseName and idx.DatasetName = ds.DatasetName
+ order by idx.IndexName
+)
+where ds.DataverseName like "test%"
+order by ds.DataverseName, ds.DatasetName;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp
index 4dc4e3b..08a41b5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp
@@ -18,7 +18,6 @@
*/
select DataverseName as CanonicalName,
- decode_dataverse_display_name(DataverseName) as DisplayName,
decode_dataverse_name(DataverseName) as NameParts
from Metadata.`Dataverse`
order by DataverseName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
index 93f8aec..19765bd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
@@ -1 +1 @@
-[ 907, 9.07, "907", 9.07, true, 15706, 621173594 ]
+null
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm
new file mode 100644
index 0000000..19765bd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm
@@ -0,0 +1 @@
+null
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.3.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.1.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.4.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.4.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm
new file mode 100644
index 0000000..a72c46c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm
@@ -0,0 +1 @@
+{ "missing": true, "missing_nullcall": "missing", "null": true, "null_nullcall": "null", "boolean": "boolean", "string": "string" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm
index 85cf5c5..28c74ac 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm
@@ -2,9 +2,9 @@
-- DISTRIBUTE_RESULT |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |LOCAL|
- aggregate [$$202] <- [agg-sql-sum($$235)]
+ aggregate [$$202] <- [agg-sql-sum($$231)]
-- AGGREGATE |LOCAL|
- aggregate [$$235] <- [agg-sql-count(1)]
+ aggregate [$$231] <- [agg-sql-count(1)]
-- AGGREGATE |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm
index f830e9b..f8a800a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm
@@ -2,9 +2,9 @@
-- DISTRIBUTE_RESULT |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |LOCAL|
- aggregate [$$180] <- [agg-sql-sum($$209)]
+ aggregate [$$180] <- [agg-sql-sum($$205)]
-- AGGREGATE |LOCAL|
- aggregate [$$209] <- [agg-sql-count(1)]
+ aggregate [$$205] <- [agg-sql-count(1)]
-- AGGREGATE |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm
new file mode 100644
index 0000000..7065d25
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm
@@ -0,0 +1,4 @@
+{ "DataverseName": "test1", "DatasetName": "a", "IsAnonymous": true, "indexes": [ { "IndexName": "a" }, { "IndexName": "ia2" }, { "IndexName": "ia3" } ] }
+{ "DataverseName": "test1", "DatasetName": "b", "IsAnonymous": true, "indexes": [ { "IndexName": "b" }, { "IndexName": "ib2" }, { "IndexName": "ib3" } ] }
+{ "DataverseName": "test2", "DatasetName": "a", "IsAnonymous": true, "indexes": [ { "IndexName": "a" }, { "IndexName": "ia2" }, { "IndexName": "ia3" } ] }
+{ "DataverseName": "test2", "DatasetName": "b", "IsAnonymous": true, "indexes": [ { "IndexName": "b" }, { "IndexName": "ib2" }, { "IndexName": "ib3" } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
index 7edd3eb..805247c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
@@ -1,6 +1,6 @@
-{ "CanonicalName": "A", "DisplayName": "A", "NameParts": [ "A" ] }
-{ "CanonicalName": "B/C", "DisplayName": "B.C", "NameParts": [ "B", "C" ] }
-{ "CanonicalName": "C.D.E", "DisplayName": "`C.D.E`", "NameParts": [ "C.D.E" ] }
-{ "CanonicalName": "Default", "DisplayName": "Default", "NameParts": [ "Default" ] }
-{ "CanonicalName": "Metadata", "DisplayName": "Metadata", "NameParts": [ "Metadata" ] }
-{ "CanonicalName": "a-A/b_B/c$C/z.Z", "DisplayName": "`a-A`.b_B.c$C.`z.Z`", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
+{ "CanonicalName": "A", "NameParts": [ "A" ] }
+{ "CanonicalName": "B/C", "NameParts": [ "B", "C" ] }
+{ "CanonicalName": "C.D.E", "NameParts": [ "C.D.E" ] }
+{ "CanonicalName": "Default", "NameParts": [ "Default" ] }
+{ "CanonicalName": "Metadata", "NameParts": [ "Metadata" ] }
+{ "CanonicalName": "a-A/b_B/c$C/z.Z", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm
index 1b39645..a809a8e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm
@@ -6,7 +6,7 @@
-- STREAM_LIMIT |UNPARTITIONED|
exchange
-- RANDOM_MERGE_EXCHANGE |PARTITIONED|
- union ($$151, $$310, $$t)
+ union ($$151, $$178, $$t)
-- UNION_ALL |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -58,7 +58,7 @@
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- union ($$345, $$356, $$310)
+ union ($$345, $$354, $$178)
-- UNION_ALL |PARTITIONED|
exchange
-- RANDOM_PARTITION_EXCHANGE |PARTITIONED|
@@ -84,9 +84,9 @@
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
exchange
-- RANDOM_PARTITION_EXCHANGE |PARTITIONED|
- project ([$$356])
+ project ([$$354])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$356] <- [{"two": $$186}]
+ assign [$$354] <- [{"two": $$186}]
-- ASSIGN |PARTITIONED|
limit 4
-- STREAM_LIMIT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index c6bad70..fca1590 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -143,11 +143,11 @@
<compilation-unit name="common/malformed-json">
<placeholder name="adapter" value="AZUREBLOB" />
<output-dir compare="Text">common/malformed-json</output-dir>
- <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field field: Duplicate field 'field'</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field field: Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field field2: Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field 'field': Duplicate field 'field'</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field 'field': Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field 'field2': Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 6557230..b354e65 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -92,11 +92,11 @@
<compilation-unit name="common/malformed-json">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">common/malformed-json</output-dir>
- <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field field: Duplicate field 'field'</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field field: Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field field2: Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field 'field': Duplicate field 'field'</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field 'field': Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field 'field2': Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
@@ -148,6 +148,13 @@
<output-dir compare="Text">non-s3-region</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="anonymous_no_auth">
+ <output-dir compare="Text">anonymous_no_auth</output-dir>
+ <expected-error>ASX3119: Parameter 'secretAccessKey' is required if 'accessKeyId' is provided</expected-error>
+ <expected-error>ASX3119: Parameter 'accessKeyId' is required if 'secretAccessKey' is provided</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/common">
<compilation-unit name="query-with-limit-plan">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml
index 92b5e32..d02647d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml
@@ -49,17 +49,17 @@
<output-dir compare="Text">common/csv-warnings</output-dir>
<expected-warn>Parsing error at data_dir/no_h_missing_fields.csv line 2 field 3: some fields are missing</expected-warn>
<expected-warn>Parsing error at data_dir/no_h_no_closing_q.csv line 2 field 0: malformed input record ended abruptly</expected-warn>
- <expected-warn>Parsing error at line 2 field 0: malformed input record ended abruptly</expected-warn>
+ <expected-warn>Parsing error at line 2 field 0: malformed input record ended abruptly</expected-warn>
- <expected-warn>Parsing error at line 5 field 3: invalid value</expected-warn>
- <expected-warn>Parsing error at line 2 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 11 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 3 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 4 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 7 field 7: invalid value</expected-warn>
- <expected-warn>Parsing error at line 13 field 7: invalid value</expected-warn>
- <expected-warn>Parsing error at line 12 field 3: invalid value</expected-warn>
- <expected-warn>Parsing error at line 9 field 6: a quote should be in the beginning</expected-warn>
+ <expected-warn>Parsing error at line 5 field 3: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 2 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 11 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 3 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 4 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 7 field 7: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 13 field 7: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 12 field 3: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 9 field 6: a quote should be in the beginning</expected-warn>
<expected-warn>Parsing error at data_dir/h_invalid_values.csv line 5 field 3: invalid value</expected-warn>
<expected-warn>Parsing error at data_dir/h_invalid_values.csv line 2 field 1: invalid value</expected-warn>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 28cbabb..6c67216 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -42,6 +42,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
+ <compilation-unit name="type_name">
+ <output-dir compare="Text">type_name</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
<compilation-unit name="type_validation">
<output-dir compare="Text">type_validation</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index f8f2597..5d1f9fd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -6846,6 +6846,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2886">
+ <output-dir compare="Text">query-ASTERIXDB-2886</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="unsupported_parameter">
<output-dir compare="Text">none</output-dir>
<expected-error>Query parameter compiler.joinmem is not supported</expected-error>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
index 7b737a0..b03af55 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
@@ -19,6 +19,7 @@
package org.apache.asterix.common.api;
import java.util.List;
+import java.util.Set;
import java.util.function.Predicate;
import org.apache.asterix.common.context.DatasetInfo;
@@ -124,6 +125,14 @@
List<IVirtualBufferCache> getVirtualBufferCaches(int datasetId, int ioDeviceNum);
/**
+ * Attempts to close the datasets in {@code datasetsToClose}
+ *
+ * @param datasetsToClose
+ * @throws HyracksDataException
+ */
+ void closeDatasets(Set<Integer> datasetsToClose) throws HyracksDataException;
+
+ /**
* Flushes then closes all open datasets
*/
void closeAllDatasets() throws HyracksDataException;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index b26220d..b2f4034 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -25,6 +25,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
@@ -452,6 +453,16 @@
}
@Override
+ public synchronized void closeDatasets(Set<Integer> datasetsToClose) throws HyracksDataException {
+ ArrayList<DatasetResource> openDatasets = new ArrayList<>(datasets.values());
+ for (DatasetResource dsr : openDatasets) {
+ if (dsr.isOpen() && datasetsToClose.contains(dsr.getDatasetID())) {
+ closeDataset(dsr);
+ }
+ }
+ }
+
+ @Override
public synchronized void closeAllDatasets() throws HyracksDataException {
ArrayList<DatasetResource> openDatasets = new ArrayList<>(datasets.values());
for (DatasetResource dsr : openDatasets) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 1be99f0..c87cb7f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -313,7 +313,7 @@
PARSER_ADM_DATA_PARSER_CAST_ERROR(3072),
PARSER_ADM_DATA_PARSER_CONSTRUCTOR_MISSING_DESERIALIZER(3073),
PARSER_ADM_DATA_PARSER_WRONG_INSTANCE(3074),
- PARSER_TWEET_PARSER_CLOSED_FIELD_NULL(3075),
+ PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL(3075),
UTIL_FILE_SYSTEM_WATCHER_NO_FILES_FOUND(3076),
UTIL_LOCAL_FILE_SYSTEM_UTILS_PATH_NOT_FOUND(3077),
UTIL_HDFS_UTILS_CANNOT_OBTAIN_HDFS_SCHEDULER(3078),
@@ -354,6 +354,8 @@
INPUT_DECODE_FAILURE(3116),
FAILED_TO_PARSE_MALFORMED_LOG_RECORD(3117),
ACTIVE_ENTITY_NOT_RUNNING(3118),
+ REQUIRED_PARAM_IF_PARAM_IS_PRESENT(3119),
+ PARSER_DATA_PARSER_UNEXPECTED_TOKEN(3120),
// Lifecycle management errors
DUPLICATE_PARTITION_ID(4000),
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
index bfe7963..8a5f34e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
@@ -85,17 +85,6 @@
long getLocalMinFirstLSN() throws HyracksDataException;
/**
- * Replay the logs that belong to the passed {@code partitions} starting from the {@code lowWaterMarkLSN}
- *
- * @param partitions
- * @param lowWaterMarkLSN
- * @throws IOException
- * @throws ACIDException
- */
- void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN)
- throws IOException, ACIDException;
-
- /**
* Creates a temporary file to be used during recovery
*
* @param txnId
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 591fa9a..f7c6a77 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -359,6 +359,8 @@
3116 = Failed to decode input
3117 = Failed to parse record, malformed log record
3118 = Active Entity %1$s is not running (it is %2$s)
+3119 = Parameter '%1$s' is required if '%2$s' is provided
+3120 = Unexpected token %s: was expecting %s
# Lifecycle management errors
4000 = Partition id %1$s for node %2$s already in use by node %3$s
diff --git a/asterixdb/asterix-doc/src/main/user-defined_function/udf.md b/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
index fe72789..71e0fa2 100644
--- a/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
+++ b/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
@@ -19,7 +19,7 @@
## <a name="introduction">Introduction</a>
-Apache AsterixDB supports three languages for writing user-defined functions (UDFs): SQL++, Java and Python.
+Apache AsterixDB supports three languages for writing user-defined functions (UDFs): SQL++, Java, and Python
A user can encapsulate data processing logic into a UDF and invoke it
later repeatedly. For SQL++ functions, a user can refer to [SQL++ Functions](sqlpp/manual.html#Functions)
for their usages. This document will focus on UDFs in languages other than SQL++
@@ -27,8 +27,10 @@
## <a name="authentication">Endpoints and Authentication</a>
-The UDF endpoint is not enabled by default until authentication has been configured properly. To enable it, we
-will need to set the path to the credential file and populate it with our username and password.
+The UDF API endpoint used to deploy functions is not enabled by default until authentication has been configured properly.
+Even if the endpoint is enabled, it is only accessible on the loopback interface on each NC to restrict access.
+
+To enable it, we need to set the path to the credential file and populate it with our username and password.
The credential file is a simple `/etc/passwd` style text file with usernames and corresponding `bcrypt` hashed and salted
passwords. You can populate this on your own if you would like, but the `asterixhelper` utility can write the entries as
@@ -50,9 +52,7 @@
## <a name="installingUDF">Installing a Java UDF Library</a>
To install a UDF package to the cluster, we need to send a Multipart Form-data HTTP request to the `/admin/udf` endpoint
-of the CC at the normal API port (`19004` by default). The request should use HTTP Basic authentication. This means your
-credentials will *not* be obfuscated or encrypted *in any way*, so submit to this endpoint over localhost or a network
-where you know your traffic is safe from eavesdropping. Any suitable tool will do, but for the example here I will use
+of the CC at the normal API port (`19004` by default). Any suitable tool will do, but for the example here I will use
`curl` which is widely available.
For example, to install a library with the following criteria:
@@ -65,7 +65,7 @@
we would execute
- curl -v -u admin:admin -X POST -F 'data=@./lib.zip' localhost:19004/admin/udf/udfs/testlib
+ curl -v -u admin:admin -X POST -F 'data=@./lib.zip' -F 'type=java' localhost:19004/admin/udf/udfs/testlib
Any response other than `200` indicates an error in deployment.
@@ -125,7 +125,7 @@
Then, deploy it the same as the Java UDF was, with the library name `pylib` in `udfs` dataverse
- curl -v -u admin:admin -X POST -F 'data=@./lib.pyz' localhost:19002/admin/udf/udfs/pylib
+ curl -v -u admin:admin -X POST -F 'data=@./lib.pyz' -F 'type=python' localhost:19002/admin/udf/udfs/pylib
With the library deployed, we can define a function within it for use. For example, to expose the Python function
`sentiment` in the module `sentiment_mod` in the class `sent_model`, the `CREATE FUNCTION` would be as follows
@@ -137,14 +137,14 @@
AS "sentiment_mod", "sent_model.sentiment" AT pylib;
By default, AsterixDB will treat all external functions as deterministic. It means the function must return the same
-result for the same input, irrespective of when or how many times the function is called on that input.
-This particular function behaves the same on each input, so it satisfies the deterministic property.
+result for the same input, irrespective of when or how many times the function is called on that input.
+This particular function behaves the same on each input, so it satisfies the deterministic property.
This enables better optimization of queries including this function.
-If a function is not deterministic then it should be declared as such by using `WITH` sub-clause:
+If a function is not deterministic then it should be declared as such by using a `WITH` sub-clause:
USE udfs;
- CREATE FUNCTION sentiment(a)
+ CREATE FUNCTION sentiment(text)
AS "sentiment_mod", "sent_model.sentiment" AT pylib
WITH { "deterministic": false }
@@ -161,6 +161,43 @@
SELECT t.msg as msg, sentiment(t.msg) as sentiment
FROM Tweets t;
+## <a name="pytpes">Python Type Mappings</a>
+
+Currently only a subset of AsterixDB types are supported in Python UDFs. The supported types are as follows:
+
+- Integer types (int8,16,32,64)
+- Floating point types (float, double)
+- String
+- Boolean
+- Arrays, Sets (cast to lists)
+- Objects (cast to dict)
+
+Unsupported types can be cast to these in SQL++ first in order to be passed to a Python UDF
+
+## <a name="execution">Execution Model For UDFs</a>
+
+AsterixDB queries are deployed across the cluster as Hyracks jobs. A Hyracks job has a lifecycle that can be simplified
+for the purposes of UDFs to
+ - A pre-run phase which allocates resources, `open`
+ - The time during which the job has data flowing through it, `nextFrame`
+ - Cleanup and shutdown in `close`.
+
+If a SQL++ function is defined as a member of a class in the library, the class will be instantiated
+during `open`. The class will exist in memory for the lifetime of the query. Therefore if your function needs to reference
+files or other data that would be costly to load per-call, making it a member variable that is initialized in the constructor
+of the object will greatly increase the performance of the SQL++ function.
+
+For each function invoked during a query, there will be an independent instance of the function per data partition. This
+means that the function must not assume there is any global state or that it can assume things about the layout
+of the data. The execution of the function will be parallel to the same degree as the level of data parallelism in the
+cluster.
+
+After initialization, the function bound in the SQL++ function definition is called once per tuple during the query
+execution (i.e. `nextFrame`). Unless the function specifies `null-call` in the `WITH` clause, `NULL` values will be
+skipped.
+
+At the close of the query, the function is torn down and not re-used in any way. All functions should assume that
+nothing will persist in-memory outside of the lifetime of a query, and any behavior contrary to this is undefined.
## <a id="UDFOnFeeds">Attaching a UDF on Data Feeds</a>
@@ -245,7 +282,7 @@
functions declared with the library are removed. First we'll drop the function we declared earlier:
USE udfs;
- DROP FUNCTION mysum@2;
+ DROP FUNCTION mysum(a,b);
Then issue the proper `DELETE` request
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
index 94d9e6e..bd422ef 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
@@ -35,6 +35,7 @@
protected final ArrayTupleBuilder tb;
protected final FeedLogManager feedLogManager;
protected boolean flushing;
+ protected long incomingRecordsCount = 0;
public AbstractFeedDataFlowController(IHyracksTaskContext ctx, FeedLogManager feedLogManager, int numOfFields) {
this.feedLogManager = feedLogManager;
@@ -50,6 +51,10 @@
flushing = false;
}
+ public long getIncomingRecordsCount() {
+ return incomingRecordsCount;
+ }
+
public abstract String getStats();
@Override
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 8cec5de..4279ebd 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -57,7 +57,6 @@
protected final AtomicBoolean closed = new AtomicBoolean(false);
protected static final long INTERVAL = 1000;
protected State state = State.CREATED;
- protected long incomingRecordsCount = 0;
protected long failedRecordsCount = 0;
public FeedRecordDataFlowController(IHyracksTaskContext ctx, FeedLogManager feedLogManager, int numOfOutputFields,
@@ -267,6 +266,10 @@
return dataParser;
}
+ public long getFailedRecordsCount() {
+ return failedRecordsCount;
+ }
+
@Override
public String getStats() {
String readerStats = recordReader.getStats();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
index b42e6de..9f4a3b0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
@@ -32,7 +32,6 @@
private final IStreamDataParser dataParser;
private final AsterixInputStream stream;
- protected long incomingRecordsCount = 0;
public FeedStreamDataFlowController(IHyracksTaskContext ctx, FeedLogManager feedLogManager,
IStreamDataParser streamParser, AsterixInputStream inputStream) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
index 3a502d0..4c253bc 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
@@ -29,10 +29,10 @@
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.asterix.external.util.ParseUtil;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.util.ParseUtil;
public class QuotedLineRecordReader extends LineRecordReader {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
index 2ff5cfa..0e23e46 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
@@ -40,10 +40,10 @@
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.asterix.external.util.ParseUtil;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.util.ParseUtil;
public class SemiStructuredRecordReader extends StreamRecordReader {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
index a8d246e..4aa5f42 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
@@ -19,7 +19,7 @@
package org.apache.asterix.external.library;
-import java.io.IOException;
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
@@ -27,7 +27,9 @@
import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionFactory;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -78,23 +80,36 @@
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
try {
- setArguments(tuple);
- resultBuffer.reset();
- externalFunctionInstance.evaluate(functionHelper);
- if (!functionHelper.isValidResult()) {
- throw new RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
+ boolean nullCall = finfo.getNullCall();
+ boolean hasNullArg = false;
+ for (int i = 0; i < argEvals.length; i++) {
+ argEvals[i].evaluate(tuple, inputVal);
+ if (!nullCall) {
+ byte[] inputValBytes = inputVal.getByteArray();
+ int inputValStartOffset = inputVal.getStartOffset();
+ ATypeTag typeTag = ATYPETAGDESERIALIZER.deserialize(inputValBytes[inputValStartOffset]);
+ if (typeTag == ATypeTag.MISSING) {
+ PointableHelper.setMissing(result);
+ return;
+ } else if (typeTag == ATypeTag.NULL) {
+ hasNullArg = true;
+ }
+ }
+ functionHelper.setArgument(i, inputVal);
}
- result.set(resultBuffer.getByteArray(), resultBuffer.getStartOffset(), resultBuffer.getLength());
- functionHelper.reset();
+ if (!nullCall && hasNullArg) {
+ PointableHelper.setNull(result);
+ } else {
+ resultBuffer.reset();
+ externalFunctionInstance.evaluate(functionHelper);
+ if (!functionHelper.isValidResult()) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
+ }
+ result.set(resultBuffer.getByteArray(), resultBuffer.getStartOffset(), resultBuffer.getLength());
+ functionHelper.reset();
+ }
} catch (Exception e) {
throw HyracksDataException.create(e);
}
}
-
- public void setArguments(IFrameTupleReference tuple) throws IOException {
- for (int i = 0; i < argEvals.length; i++) {
- argEvals[i].evaluate(tuple, inputVal);
- functionHelper.setArgument(i, inputVal);
- }
- }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
index e664f47..7c860a2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.library;
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import static org.msgpack.core.MessagePack.Code.FIXARRAY_PREFIX;
import java.io.DataOutput;
@@ -28,6 +29,7 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.library.msgpack.MessageUnpackerToADM;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
@@ -76,9 +78,11 @@
for (int i = 0; i < argValues.length; i++) {
argValues[i] = VoidPointable.FACTORY.createPointable();
}
- //TODO: these should be dynamic
- this.argHolder = ByteBuffer.wrap(new byte[Short.MAX_VALUE * 2]);
- this.outputWrapper = ByteBuffer.wrap(new byte[Short.MAX_VALUE * 2]);
+ //TODO: these should be dynamic. this static size picking is a temporary bodge until this works like
+ // v-size frames do or these construction buffers are removed entirely
+ int maxArgSz = ExternalDataUtils.getArgBufferSize();
+ this.argHolder = ByteBuffer.wrap(new byte[maxArgSz]);
+ this.outputWrapper = ByteBuffer.wrap(new byte[maxArgSz]);
this.evaluatorContext = ctx;
this.sourceLocation = sourceLoc;
this.unpackerInput = new ArrayBufferInput(new byte[0]);
@@ -88,17 +92,31 @@
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
argHolder.clear();
+ boolean nullCall = finfo.getNullCall();
+ boolean hasNullArg = false;
for (int i = 0, ln = argEvals.length; i < ln; i++) {
argEvals[i].evaluate(tuple, argValues[i]);
- if (!finfo.getNullCall() && PointableHelper.checkAndSetMissingOrNull(result, argValues[i])) {
- return;
+ if (!nullCall) {
+ byte[] argBytes = argValues[i].getByteArray();
+ int argStart = argValues[i].getStartOffset();
+ ATypeTag argType = ATYPETAGDESERIALIZER.deserialize(argBytes[argStart]);
+ if (argType == ATypeTag.MISSING) {
+ PointableHelper.setMissing(result);
+ return;
+ } else if (argType == ATypeTag.NULL) {
+ hasNullArg = true;
+ }
}
try {
- PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, finfo.getNullCall());
+ PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, nullCall);
} catch (IOException e) {
throw new HyracksDataException("Error evaluating Python UDF", e);
}
}
+ if (!nullCall && hasNullArg) {
+ PointableHelper.setNull(result);
+ return;
+ }
try {
ByteBuffer res = libraryEvaluator.callPython(fnId, argHolder, argTypes.length);
resultBuffer.reset();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
index 39e480a..593bac6 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
@@ -36,6 +36,7 @@
import org.apache.asterix.external.library.PythonLibraryEvaluator;
import org.apache.asterix.external.library.PythonLibraryEvaluatorFactory;
import org.apache.asterix.external.library.msgpack.MessageUnpackerToADM;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.om.functions.IExternalFunctionDescriptor;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -61,6 +62,8 @@
private final IExternalFunctionDescriptor[] fnDescs;
private final int[][] fnArgColumns;
+ private int rpcBufferSize;
+
public ExternalAssignBatchRuntimeFactory(int[] outColumns, IExternalFunctionDescriptor[] fnDescs,
int[][] fnArgColumns, int[] projectionList) {
super(projectionList);
@@ -73,6 +76,9 @@
public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(IHyracksTaskContext ctx) {
final int[] projectionToOutColumns = new int[projectionList.length];
+ //this is a temporary bodge. these buffers need to work like vsize frames, or be absent entirely
+ int maxArgSz = ExternalDataUtils.getArgBufferSize();
+ rpcBufferSize = ExternalDataUtils.roundUpToNearestFrameSize(maxArgSz, ctx.getInitialFrameSize());
for (int j = 0; j < projectionList.length; j++) {
projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
}
@@ -110,14 +116,14 @@
}
argHolders = new ArrayList<>(fnArgColumns.length);
for (int i = 0; i < fnArgColumns.length; i++) {
- argHolders.add(ctx.allocateFrame());
+ argHolders.add(ctx.allocateFrame(rpcBufferSize));
}
outputWrapper = ctx.allocateFrame();
nullCalls = new ATypeTag[argHolders.size()][0];
numCalls = new int[fnArgColumns.length];
batchResults = new ArrayList<>(argHolders.size());
for (int i = 0; i < argHolders.size(); i++) {
- batchResults.add(new Pair<>(ctx.allocateFrame(), new Counter(-1)));
+ batchResults.add(new Pair<>(ctx.allocateFrame(rpcBufferSize), new Counter(-1)));
}
unpackerInput = new ArrayBufferInput(new byte[0]);
unpacker = MessagePack.newDefaultUnpacker(unpackerInput);
@@ -230,7 +236,8 @@
if (columnResult != null) {
Pair<ByteBuffer, Counter> resultholder = batchResults.get(argHolderIdx);
if (resultholder.getFirst().capacity() < columnResult.capacity()) {
- resultholder.setFirst(ctx.allocateFrame(columnResult.capacity()));
+ resultholder.setFirst(ctx.allocateFrame(ExternalDataUtils.roundUpToNearestFrameSize(
+ columnResult.capacity(), ctx.getInitialFrameSize())));
}
ByteBuffer resultBuf = resultholder.getFirst();
resultBuf.clear();
@@ -262,6 +269,12 @@
outputWrapper.clear();
outputWrapper.position(0);
Pair<ByteBuffer, Counter> result = batchResults.get(k);
+ if (result.getFirst() != null) {
+ if (result.getFirst().capacity() > outputWrapper.capacity()) {
+ outputWrapper = ctx.allocateFrame(ExternalDataUtils.roundUpToNearestFrameSize(
+ outputWrapper.capacity(), ctx.getInitialFrameSize()));
+ }
+ }
int start = outputWrapper.arrayOffset();
ATypeTag functionCalled = nullCalls[k][i];
if (functionCalled == ATypeTag.TYPE) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
new file mode 100644
index 0000000..2d20cc2
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.parser;
+
+import static org.apache.hyracks.api.exceptions.ErrorCode.PARSING_ERROR;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
+
+import org.apache.asterix.builders.IARecordBuilder;
+import org.apache.asterix.builders.IAsterixListBuilder;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.external.parser.jackson.ADMToken;
+import org.apache.asterix.external.parser.jackson.GeometryCoParser;
+import org.apache.asterix.external.parser.jackson.ParserContext;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.ANull;
+import org.apache.asterix.om.base.AUnorderedList;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.asterix.runtime.exceptions.UnsupportedTypeException;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.ExceptionUtils;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.util.ParseUtil;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonStreamContext;
+import com.fasterxml.jackson.core.JsonToken;
+
+/**
+ * JSON format parser using Jackson parser.
+ */
+public abstract class AbstractJsonDataParser extends AbstractNestedDataParser<ADMToken> {
+
+ protected final ParserContext parserContext;
+ protected final JsonFactory jsonFactory;
+ protected final ARecordType rootType;
+ protected final GeometryCoParser geometryCoParser;
+ protected Supplier<String> dataSourceName = ExternalDataConstants.EMPTY_STRING;
+ protected LongSupplier lineNumber = ExternalDataConstants.NO_LINES;
+
+ protected JsonParser jsonParser;
+
+ /**
+ * Initialize JSONDataParser with GeometryCoParser
+ *
+ * @param recordType
+ * defined type.
+ * @param jsonFactory
+ * Jackson JSON parser factory.
+ */
+ public AbstractJsonDataParser(ARecordType recordType, JsonFactory jsonFactory) {
+ // recordType currently cannot be null, however this is to guarantee for any future changes.
+ this.rootType = recordType != null ? recordType : RecordUtil.FULLY_OPEN_RECORD_TYPE;
+ this.jsonFactory = jsonFactory;
+ //GeometryCoParser to parse GeoJSON objects to AsterixDB internal spatial types.
+ geometryCoParser = new GeometryCoParser(jsonParser);
+ parserContext = new ParserContext();
+ }
+
+ /*
+ ****************************************************
+ * Public methods
+ ****************************************************
+ */
+
+ public boolean parseAnyValue(DataOutput out) throws HyracksDataException {
+ try {
+ if (nextToken() == ADMToken.EOF) {
+ return false;
+ }
+ parseValue(BuiltinType.ANY, out);
+ return true;
+ } catch (IOException e) {
+ throw new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
+ }
+ }
+
+ /*
+ ****************************************************
+ * Abstract method implementation
+ ****************************************************
+ */
+
+ /**
+ * Jackson token to ADM token mapper
+ */
+ @Override
+ protected final ADMToken advanceToNextToken() throws IOException {
+ final JsonToken jsonToken = jsonParser.nextToken();
+ if (jsonToken == null) {
+ return ADMToken.EOF;
+ }
+ ADMToken token;
+ switch (jsonToken) {
+ case VALUE_FALSE:
+ token = ADMToken.FALSE;
+ break;
+ case VALUE_TRUE:
+ token = ADMToken.TRUE;
+ break;
+ case VALUE_STRING:
+ token = ADMToken.STRING;
+ break;
+ case VALUE_NULL:
+ token = ADMToken.NULL;
+ break;
+ case VALUE_NUMBER_FLOAT:
+ token = ADMToken.DOUBLE;
+ break;
+ case VALUE_NUMBER_INT:
+ token = ADMToken.INT;
+ break;
+ case START_OBJECT:
+ token = ADMToken.OBJECT_START;
+ break;
+ case END_OBJECT:
+ token = ADMToken.OBJECT_END;
+ break;
+ case START_ARRAY:
+ token = ADMToken.ARRAY_START;
+ break;
+ case END_ARRAY:
+ token = ADMToken.ARRAY_END;
+ break;
+ case FIELD_NAME:
+ token = ADMToken.FIELD_NAME;
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
+ }
+
+ return token;
+ }
+ /*
+ ****************************************************
+ * Overridden methods
+ ****************************************************
+ */
+
+ /**
+ * In the case of JSON, we can parse GeoJSON objects as internal AsterixDB spatial types.
+ */
+ @Override
+ protected boolean isConvertable(ATypeTag parsedTypeTag, ATypeTag definedTypeTag) {
+ if (parsedTypeTag == ATypeTag.OBJECT && (definedTypeTag == ATypeTag.POINT || definedTypeTag == ATypeTag.LINE
+ || definedTypeTag == ATypeTag.POLYGON)) {
+ return true;
+ }
+ return super.isConvertable(parsedTypeTag, definedTypeTag);
+ }
+
+ /*
+ ****************************************************
+ * Complex types parsers
+ ****************************************************
+ */
+
+ @Override
+ protected final void parseObject(ARecordType recordType, DataOutput out) throws IOException {
+ final IMutableValueStorage valueBuffer = parserContext.enterObject();
+ final IARecordBuilder objectBuilder = parserContext.getObjectBuilder(recordType);
+ final BitSet nullBitMap = parserContext.getNullBitmap(recordType.getFieldTypes().length);
+ while (nextToken() != ADMToken.OBJECT_END) {
+ /*
+ * Jackson parser calls String.intern() for field names (if enabled).
+ * Calling getCurrentName() will not create multiple objects.
+ */
+ final String fieldName = jsonParser.getCurrentName();
+ final int fieldIndex = recordType.getFieldIndex(fieldName);
+
+ if (!recordType.isOpen() && fieldIndex < 0) {
+ throw new RuntimeDataException(ErrorCode.PARSER_ADM_DATA_PARSER_EXTRA_FIELD_IN_CLOSED_RECORD,
+ fieldName);
+ }
+ valueBuffer.reset();
+ nextToken();
+
+ if (fieldIndex < 0) {
+ //field is not defined and the type is open
+ parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
+ objectBuilder.addField(parserContext.getSerializedFieldName(fieldName), valueBuffer);
+ } else {
+ //field is defined
+ final IAType fieldType = recordType.getFieldType(fieldName);
+
+ //fail fast if the current field is not nullable
+ if (currentToken() == ADMToken.NULL && !isNullableType(fieldType)) {
+ throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL, fieldName);
+ }
+
+ nullBitMap.set(fieldIndex);
+ parseValue(fieldType, valueBuffer.getDataOutput());
+ objectBuilder.addField(fieldIndex, valueBuffer);
+ }
+ }
+
+ /*
+ * Check for any possible missed values for a defined (non-nullable) type.
+ * Throws exception if there is a violation
+ */
+ if (nullBitMap != null) {
+ checkOptionalConstraints(recordType, nullBitMap);
+ }
+ parserContext.exitObject(valueBuffer, nullBitMap, objectBuilder);
+ objectBuilder.write(out, true);
+ }
+
+ /**
+ * Geometry in GeoJSON is an object
+ *
+ * @param typeTag
+ * geometry typeTag
+ * @param out
+ * @throws IOException
+ */
+ private void parseGeometry(ATypeTag typeTag, DataOutput out) throws IOException {
+ //Start the co-parser
+ geometryCoParser.starGeometry();
+ while (nextToken() != ADMToken.OBJECT_END) {
+ if (currentToken() == ADMToken.FIELD_NAME) {
+ geometryCoParser.checkFieldName(jsonParser.getCurrentName());
+ } else if (!geometryCoParser.checkValue(currentToken())) {
+ throw new IOException(geometryCoParser.getErrorMessage());
+ }
+ }
+
+ geometryCoParser.serialize(typeTag, out);
+ }
+
+ @Override
+ protected final void parseArray(AOrderedListType listType, DataOutput out) throws IOException {
+ parseCollection(listType, ADMToken.ARRAY_END, out);
+ }
+
+ @Override
+ protected void parseMultiset(AUnorderedList listType, DataOutput out) throws IOException {
+ throw new UnsupportedTypeException("JSON parser", ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ }
+
+ protected final void parseCollection(AbstractCollectionType collectionType, ADMToken endToken, DataOutput out)
+ throws IOException {
+ final IMutableValueStorage valueBuffer = parserContext.enterCollection();
+ final IAsterixListBuilder arrayBuilder = parserContext.getCollectionBuilder(collectionType);
+ final boolean isOpen = collectionType.getItemType().getTypeTag() == ATypeTag.ANY;
+ while (nextToken() != endToken) {
+ valueBuffer.reset();
+ if (isOpen) {
+ parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
+ } else {
+ //fail fast if current value is null
+ if (currentToken() == ADMToken.NULL) {
+ throw new RuntimeDataException(ErrorCode.PARSER_COLLECTION_ITEM_CANNOT_BE_NULL);
+ }
+ parseValue(collectionType.getItemType(), valueBuffer.getDataOutput());
+ }
+ arrayBuilder.addItem(valueBuffer);
+ }
+ parserContext.exitCollection(valueBuffer, arrayBuilder);
+ arrayBuilder.write(out, true);
+ }
+
+ /*
+ ****************************************************
+ * Value parsers and serializers
+ ****************************************************
+ */
+
+ /**
+ * Parse JSON object or GeoJSON object.
+ *
+ * @param actualType
+ * @param out
+ * @throws IOException
+ */
+ protected void parseObject(IAType actualType, DataOutput out) throws IOException {
+ if (actualType.getTypeTag() == ATypeTag.OBJECT) {
+ parseObject((ARecordType) actualType, out);
+ } else {
+ parseGeometry(actualType.getTypeTag(), out);
+ }
+ }
+
+ protected void parseValue(IAType definedType, DataOutput out) throws IOException {
+ final ATypeTag currentTypeTag = currentToken().getTypeTag();
+ /*
+ * In case of type mismatch, checkAndGetType will throw an exception.
+ */
+ final IAType actualType = checkAndGetType(definedType, currentTypeTag);
+
+ switch (currentToken()) {
+ case NULL:
+ nullSerde.serialize(ANull.NULL, out);
+ break;
+ case FALSE:
+ booleanSerde.serialize(ABoolean.FALSE, out);
+ break;
+ case TRUE:
+ booleanSerde.serialize(ABoolean.TRUE, out);
+ break;
+ case INT:
+ case DOUBLE:
+ serailizeNumeric(actualType.getTypeTag(), out);
+ break;
+ case STRING:
+ serializeString(actualType.getTypeTag(), out);
+ break;
+ case OBJECT_START:
+ parseObject(actualType, out);
+ break;
+ case ARRAY_START:
+ parseArray((AOrderedListType) actualType, out);
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.PARSE_ERROR, jsonParser.currentToken().toString());
+ }
+ }
+
+ /**
+ * Given that numeric values may underflow or overflow, an exception will be thrown.
+ *
+ * @param numericType
+ * @param out
+ * @throws IOException
+ */
+ private void serailizeNumeric(ATypeTag numericType, DataOutput out) throws IOException {
+ final ATypeTag typeToUse = numericType == ATypeTag.ANY ? currentToken().getTypeTag() : numericType;
+
+ switch (typeToUse) {
+ case BIGINT:
+ aInt64.setValue(jsonParser.getLongValue());
+ int64Serde.serialize(aInt64, out);
+ break;
+ case INTEGER:
+ aInt32.setValue(jsonParser.getIntValue());
+ int32Serde.serialize(aInt32, out);
+ break;
+ case SMALLINT:
+ aInt16.setValue(jsonParser.getShortValue());
+ int16Serde.serialize(aInt16, out);
+ break;
+ case TINYINT:
+ aInt8.setValue(jsonParser.getByteValue());
+ int8Serde.serialize(aInt8, out);
+ break;
+ case DOUBLE:
+ aDouble.setValue(jsonParser.getDoubleValue());
+ doubleSerde.serialize(aDouble, out);
+ break;
+ case FLOAT:
+ aFloat.setValue(jsonParser.getFloatValue());
+ floatSerde.serialize(aFloat, out);
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
+ }
+ }
+
+ /**
+ * Serialize the string value.
+ * TODO(wyk) avoid String objects for type STRING
+ *
+ * @param stringVariantType
+ * @param out
+ * @throws IOException
+ */
+ private void serializeString(ATypeTag stringVariantType, DataOutput out) throws IOException {
+ char[] buffer = jsonParser.getTextCharacters();
+ int begin = jsonParser.getTextOffset();
+ int len = jsonParser.getTextLength();
+ final ATypeTag typeToUse = stringVariantType == ATypeTag.ANY ? currentToken().getTypeTag() : stringVariantType;
+
+ switch (typeToUse) {
+ case STRING:
+ parseString(buffer, begin, len, out);
+ break;
+ case DATE:
+ parseDate(buffer, begin, len, out);
+ break;
+ case DATETIME:
+ parseDateTime(buffer, begin, len, out);
+ break;
+ case TIME:
+ parseTime(buffer, begin, len, out);
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
+
+ }
+ }
+
+ protected HyracksDataException createException(Exception e) {
+ if (jsonParser != null) {
+ String msg;
+ if (e instanceof JsonParseException) {
+ msg = ((JsonParseException) e).getOriginalMessage();
+ } else {
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause instanceof ParseException) {
+ msg = ((ParseException) rootCause).getOriginalMessage();
+ } else {
+ msg = ExceptionUtils.getRootCause(e).getMessage();
+ }
+ }
+ if (msg == null) {
+ msg = ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM.errorMessage();
+ }
+ long lineNum = lineNumber.getAsLong() + jsonParser.getCurrentLocation().getLineNr() - 1;
+ JsonStreamContext parsingContext = jsonParser.getParsingContext();
+ String fieldName = null;
+ while (parsingContext != null && fieldName == null) {
+ fieldName = parsingContext.getCurrentName();
+ parsingContext = parsingContext.getParent();
+ }
+ final String locationDetails = ParseUtil.asLocationDetailString(dataSourceName.get(), lineNum, fieldName);
+ return HyracksDataException.create(PARSING_ERROR, locationDetails, msg);
+ }
+ return new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java
index c6f605d..eecbb19 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java
@@ -116,7 +116,7 @@
protected void checkOptionalConstraints(ARecordType recordType, BitSet nullBitmap) throws RuntimeDataException {
for (int i = 0; i < recordType.getFieldTypes().length; i++) {
if (!nullBitmap.get(i) && !isMissableType(recordType.getFieldTypes()[i])) {
- throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL,
+ throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL,
recordType.getFieldNames()[i]);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
index 60e6e77..590f51d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
@@ -38,7 +38,6 @@
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IStreamDataParser;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ParseUtil;
import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
@@ -52,6 +51,7 @@
import org.apache.hyracks.dataflow.common.data.parsers.IValueParser;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.std.file.FieldCursorForDelimitedDataParser;
+import org.apache.hyracks.util.ParseUtil;
public class DelimitedDataParser extends AbstractDataParser implements IStreamDataParser, IRecordDataParser<char[]> {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
index b2036c0..b2cffa9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
@@ -18,64 +18,36 @@
*/
package org.apache.asterix.external.parser;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARSER_DATA_PARSER_UNEXPECTED_TOKEN;
+
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
-import java.util.BitSet;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
-import org.apache.asterix.builders.IARecordBuilder;
-import org.apache.asterix.builders.IAsterixListBuilder;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IStreamDataParser;
import org.apache.asterix.external.parser.jackson.ADMToken;
-import org.apache.asterix.external.parser.jackson.GeometryCoParser;
-import org.apache.asterix.external.parser.jackson.ParserContext;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.base.ANull;
-import org.apache.asterix.om.base.AUnorderedList;
-import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.AbstractCollectionType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.utils.RecordUtil;
-import org.apache.asterix.runtime.exceptions.UnsupportedTypeException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.util.ExceptionUtils;
-import org.apache.hyracks.data.std.api.IMutableValueStorage;
import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonStreamContext;
-import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.TreeTraversingParser;
/**
- * JSON format parser using Jakson parser.
+ * JSON format parser using Jackson parser.
*/
-public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
- implements IStreamDataParser, IRecordDataParser<char[]> {
-
- protected final ParserContext parserContext;
- protected final JsonFactory jsonFactory;
- protected final ARecordType rootType;
- protected final GeometryCoParser geometryCoParser;
- private Supplier<String> dataSourceName;
- private LongSupplier lineNumber;
-
- protected JsonParser jsonParser;
+public class JSONDataParser extends AbstractJsonDataParser implements IStreamDataParser, IRecordDataParser<char[]> {
/**
- * Initialize JSONDataParser with GeometryCoParser
+ * Initialize JSONDataParser
*
* @param recordType
* defined type.
@@ -83,40 +55,7 @@
* Jackson JSON parser factory.
*/
public JSONDataParser(ARecordType recordType, JsonFactory jsonFactory) {
- // recordType currently cannot be null, however this is to guarantee for any future changes.
- this.rootType = recordType != null ? recordType : RecordUtil.FULLY_OPEN_RECORD_TYPE;
- this.jsonFactory = jsonFactory;
- //GeometyCoParser to parse GeoJSON objects to AsterixDB internal spatial types.
- geometryCoParser = new GeometryCoParser(jsonParser);
- parserContext = new ParserContext();
- this.dataSourceName = ExternalDataConstants.EMPTY_STRING;
- this.lineNumber = ExternalDataConstants.NO_LINES;
- }
-
- /*
- ****************************************************
- * Public methods
- ****************************************************
- */
-
- @Override
- public void configure(Supplier<String> dataSourceName, LongSupplier lineNumber) {
- this.dataSourceName = dataSourceName == null ? ExternalDataConstants.EMPTY_STRING : dataSourceName;
- this.lineNumber = lineNumber == null ? ExternalDataConstants.NO_LINES : lineNumber;
- }
-
- @Override
- public final boolean parse(IRawRecord<? extends char[]> record, DataOutput out) throws HyracksDataException {
- try {
- //TODO(wyk): find a way to reset byte[] instead of creating a new parser for each record.
- jsonParser = jsonFactory.createParser(record.get(), 0, record.size());
- geometryCoParser.reset(jsonParser);
- nextToken();
- parseObject(rootType, out);
- return true;
- } catch (IOException e) {
- throw createException(e);
- }
+ super(recordType, jsonFactory);
}
@Override
@@ -134,6 +73,28 @@
}
@Override
+ public boolean parse(IRawRecord<? extends char[]> record, DataOutput out) throws HyracksDataException {
+ try {
+ //TODO(wyk): find a way to reset byte[] instead of creating a new parser for each record.
+ jsonParser = jsonFactory.createParser(record.get(), 0, record.size());
+ geometryCoParser.reset(jsonParser);
+ if (nextToken() != ADMToken.OBJECT_START) {
+ throw new ParseException(PARSER_DATA_PARSER_UNEXPECTED_TOKEN, currentToken(), ADMToken.OBJECT_START);
+ }
+ parseObject(rootType, out);
+ return true;
+ } catch (IOException e) {
+ throw createException(e);
+ }
+ }
+
+ @Override
+ public void configure(Supplier<String> dataSourceName, LongSupplier lineNumber) {
+ this.dataSourceName = dataSourceName == null ? ExternalDataConstants.EMPTY_STRING : dataSourceName;
+ this.lineNumber = lineNumber == null ? ExternalDataConstants.NO_LINES : lineNumber;
+ }
+
+ @Override
public boolean parse(DataOutput out) throws HyracksDataException {
try {
if (nextToken() == ADMToken.EOF) {
@@ -146,363 +107,9 @@
}
}
- public boolean parseAnyValue(DataOutput out) throws HyracksDataException {
- try {
- if (nextToken() == ADMToken.EOF) {
- return false;
- }
- parseValue(BuiltinType.ANY, out);
- return true;
- } catch (IOException e) {
- throw new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
- }
- }
-
@Override
public boolean reset(InputStream in) throws IOException {
setInputStream(in);
return true;
}
-
- /*
- ****************************************************
- * Abstract method implementation
- ****************************************************
- */
-
- /**
- * Jackson token to ADM token mapper
- */
- @Override
- protected final ADMToken advanceToNextToken() throws IOException {
- final JsonToken jsonToken = jsonParser.nextToken();
- if (jsonToken == null) {
- return ADMToken.EOF;
- }
- ADMToken token;
- switch (jsonToken) {
- case VALUE_FALSE:
- token = ADMToken.FALSE;
- break;
- case VALUE_TRUE:
- token = ADMToken.TRUE;
- break;
- case VALUE_STRING:
- token = ADMToken.STRING;
- break;
- case VALUE_NULL:
- token = ADMToken.NULL;
- break;
- case VALUE_NUMBER_FLOAT:
- token = ADMToken.DOUBLE;
- break;
- case VALUE_NUMBER_INT:
- token = ADMToken.INT;
- break;
- case START_OBJECT:
- token = ADMToken.OBJECT_START;
- break;
- case END_OBJECT:
- token = ADMToken.OBJECT_END;
- break;
- case START_ARRAY:
- token = ADMToken.ARRAY_START;
- break;
- case END_ARRAY:
- token = ADMToken.ARRAY_END;
- break;
- case FIELD_NAME:
- token = ADMToken.FIELD_NAME;
- break;
- default:
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
- }
-
- return token;
- }
- /*
- ****************************************************
- * Overridden methods
- ****************************************************
- */
-
- /**
- * In the case of JSON, we can parse GeoJSON objects as internal AsterixDB spatial types.
- */
- @Override
- protected boolean isConvertable(ATypeTag parsedTypeTag, ATypeTag definedTypeTag) {
- if (parsedTypeTag == ATypeTag.OBJECT && (definedTypeTag == ATypeTag.POINT || definedTypeTag == ATypeTag.LINE
- || definedTypeTag == ATypeTag.POLYGON)) {
- return true;
- }
- return super.isConvertable(parsedTypeTag, definedTypeTag);
- }
-
- /*
- ****************************************************
- * Complex types parsers
- ****************************************************
- */
-
- @Override
- protected final void parseObject(ARecordType recordType, DataOutput out) throws IOException {
- final IMutableValueStorage valueBuffer = parserContext.enterObject();
- final IARecordBuilder objectBuilder = parserContext.getObjectBuilder(recordType);
- final BitSet nullBitMap = parserContext.getNullBitmap(recordType.getFieldTypes().length);
- while (nextToken() != ADMToken.OBJECT_END) {
- /*
- * Jackson parser calls String.intern() for field names (if enabled).
- * Calling getCurrentName() will not create multiple objects.
- */
- final String fieldName = jsonParser.getCurrentName();
- final int fieldIndex = recordType.getFieldIndex(fieldName);
-
- if (!recordType.isOpen() && fieldIndex < 0) {
- throw new RuntimeDataException(ErrorCode.PARSER_ADM_DATA_PARSER_EXTRA_FIELD_IN_CLOSED_RECORD,
- fieldName);
- }
- valueBuffer.reset();
- nextToken();
-
- if (fieldIndex < 0) {
- //field is not defined and the type is open
- parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
- objectBuilder.addField(parserContext.getSerializedFieldName(fieldName), valueBuffer);
- } else {
- //field is defined
- final IAType fieldType = recordType.getFieldType(fieldName);
-
- //fail fast if the current field is not nullable
- if (currentToken() == ADMToken.NULL && !isNullableType(fieldType)) {
- throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL, fieldName);
- }
-
- nullBitMap.set(fieldIndex);
- parseValue(fieldType, valueBuffer.getDataOutput());
- objectBuilder.addField(fieldIndex, valueBuffer);
- }
- }
-
- /*
- * Check for any possible missed values for a defined (non-nullable) type.
- * Throws exception if there is a violation
- */
- if (nullBitMap != null) {
- checkOptionalConstraints(recordType, nullBitMap);
- }
- parserContext.exitObject(valueBuffer, nullBitMap, objectBuilder);
- objectBuilder.write(out, true);
- }
-
- /**
- * Geometry in GeoJSON is an object
- *
- * @param typeTag
- * geometry typeTag
- * @param out
- * @throws IOException
- */
- private void parseGeometry(ATypeTag typeTag, DataOutput out) throws IOException {
- //Start the co-parser
- geometryCoParser.starGeometry();
- while (nextToken() != ADMToken.OBJECT_END) {
- if (currentToken() == ADMToken.FIELD_NAME) {
- geometryCoParser.checkFieldName(jsonParser.getCurrentName());
- } else if (!geometryCoParser.checkValue(currentToken())) {
- throw new IOException(geometryCoParser.getErrorMessage());
- }
- }
-
- geometryCoParser.serialize(typeTag, out);
- }
-
- @Override
- protected final void parseArray(AOrderedListType listType, DataOutput out) throws IOException {
- parseCollection(listType, ADMToken.ARRAY_END, out);
- }
-
- @Override
- protected void parseMultiset(AUnorderedList listType, DataOutput out) throws IOException {
- throw new UnsupportedTypeException("JSON parser", ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
-
- protected final void parseCollection(AbstractCollectionType collectionType, ADMToken endToken, DataOutput out)
- throws IOException {
- final IMutableValueStorage valueBuffer = parserContext.enterCollection();
- final IAsterixListBuilder arrayBuilder = parserContext.getCollectionBuilder(collectionType);
- final boolean isOpen = collectionType.getItemType().getTypeTag() == ATypeTag.ANY;
- while (nextToken() != endToken) {
- valueBuffer.reset();
- if (isOpen) {
- parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
- } else {
- //fail fast if current value is null
- if (currentToken() == ADMToken.NULL) {
- throw new RuntimeDataException(ErrorCode.PARSER_COLLECTION_ITEM_CANNOT_BE_NULL);
- }
- parseValue(collectionType.getItemType(), valueBuffer.getDataOutput());
- }
- arrayBuilder.addItem(valueBuffer);
- }
- parserContext.exitCollection(valueBuffer, arrayBuilder);
- arrayBuilder.write(out, true);
- }
-
- /*
- ****************************************************
- * Value parsers and serializers
- ****************************************************
- */
-
- /**
- * Parse JSON object or GeoJSON object.
- *
- * @param actualType
- * @param out
- * @throws IOException
- */
- private void parseObject(IAType actualType, DataOutput out) throws IOException {
- if (actualType.getTypeTag() == ATypeTag.OBJECT) {
- parseObject((ARecordType) actualType, out);
- } else {
- parseGeometry(actualType.getTypeTag(), out);
- }
- }
-
- protected void parseValue(IAType definedType, DataOutput out) throws IOException {
- final ATypeTag currentTypeTag = currentToken().getTypeTag();
- /*
- * In case of type mismatch, checkAndGetType will throw an exception.
- */
- final IAType actualType = checkAndGetType(definedType, currentTypeTag);
-
- switch (currentToken()) {
- case NULL:
- nullSerde.serialize(ANull.NULL, out);
- break;
- case FALSE:
- booleanSerde.serialize(ABoolean.FALSE, out);
- break;
- case TRUE:
- booleanSerde.serialize(ABoolean.TRUE, out);
- break;
- case INT:
- case DOUBLE:
- serailizeNumeric(actualType.getTypeTag(), out);
- break;
- case STRING:
- serializeString(actualType.getTypeTag(), out);
- break;
- case OBJECT_START:
- parseObject(actualType, out);
- break;
- case ARRAY_START:
- parseArray((AOrderedListType) actualType, out);
- break;
- default:
- throw new RuntimeDataException(ErrorCode.PARSE_ERROR, jsonParser.currentToken().toString());
- }
- }
-
- /**
- * Given that numeric values may underflow or overflow, an exception will be thrown.
- *
- * @param numericType
- * @param out
- * @throws IOException
- */
- private void serailizeNumeric(ATypeTag numericType, DataOutput out) throws IOException {
- final ATypeTag typeToUse = numericType == ATypeTag.ANY ? currentToken().getTypeTag() : numericType;
-
- switch (typeToUse) {
- case BIGINT:
- aInt64.setValue(jsonParser.getLongValue());
- int64Serde.serialize(aInt64, out);
- break;
- case INTEGER:
- aInt32.setValue(jsonParser.getIntValue());
- int32Serde.serialize(aInt32, out);
- break;
- case SMALLINT:
- aInt16.setValue(jsonParser.getShortValue());
- int16Serde.serialize(aInt16, out);
- break;
- case TINYINT:
- aInt8.setValue(jsonParser.getByteValue());
- int8Serde.serialize(aInt8, out);
- break;
- case DOUBLE:
- aDouble.setValue(jsonParser.getDoubleValue());
- doubleSerde.serialize(aDouble, out);
- break;
- case FLOAT:
- aFloat.setValue(jsonParser.getFloatValue());
- floatSerde.serialize(aFloat, out);
- break;
- default:
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
- }
- }
-
- /**
- * Serialize the string value.
- * TODO(wyk) avoid String objects for type STRING
- *
- * @param stringVariantType
- * @param out
- * @throws IOException
- */
- private void serializeString(ATypeTag stringVariantType, DataOutput out) throws IOException {
- char[] buffer = jsonParser.getTextCharacters();
- int begin = jsonParser.getTextOffset();
- int len = jsonParser.getTextLength();
- final ATypeTag typeToUse = stringVariantType == ATypeTag.ANY ? currentToken().getTypeTag() : stringVariantType;
-
- switch (typeToUse) {
- case STRING:
- parseString(buffer, begin, len, out);
- break;
- case DATE:
- parseDate(buffer, begin, len, out);
- break;
- case DATETIME:
- parseDateTime(buffer, begin, len, out);
- break;
- case TIME:
- parseTime(buffer, begin, len, out);
- break;
- default:
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
-
- }
- }
-
- private HyracksDataException createException(IOException e) {
- if (jsonParser != null) {
- String msg;
- if (e instanceof JsonParseException) {
- msg = ((JsonParseException) e).getOriginalMessage();
- } else {
- msg = ExceptionUtils.getRootCause(e).getMessage();
- }
- if (msg == null) {
- msg = ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM.errorMessage();
- }
- long lineNum = lineNumber.getAsLong() + jsonParser.getCurrentLocation().getLineNr() - 1;
- JsonStreamContext parsingContext = jsonParser.getParsingContext();
- String fieldName = "N/A";
- while (parsingContext != null) {
- String currentFieldName = parsingContext.getCurrentName();
- if (currentFieldName != null) {
- fieldName = currentFieldName;
- break;
- }
- parsingContext = parsingContext.getParent();
- }
-
- return HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.PARSING_ERROR,
- dataSourceName.get(), lineNum, fieldName, msg);
- }
- return new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
- }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java
index e9f93c9..f130fb3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java
@@ -84,4 +84,8 @@
}
return msg.append(": ").append(super.getMessage()).toString();
}
+
+ public String getOriginalMessage() {
+ return super.getMessage();
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
index 4726a50..b970b04 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
@@ -199,7 +199,7 @@
DataOutput fieldOutput = fieldValueBuffer.getDataOutput();
if (obj.get(curFNames[iter1]).isNull() && !(curTypes[iter1] instanceof AUnionType)) {
if (curRecType.isClosedField(curFNames[iter1])) {
- throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL,
+ throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL,
curFNames[iter1]);
} else {
continue;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
new file mode 100644
index 0000000..ef75688
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.parser.jackson;
+
+import java.util.ArrayDeque;
+import java.util.Queue;
+
+import org.apache.asterix.om.util.container.IObjectFactory;
+
+/**
+ * Object pool for DFS traversal mode, which allows to recycle objects
+ * as soon as it is not needed.
+ */
+public abstract class AbstractObjectPool<E, T, Q> implements IObjectPool<E> {
+ private final IObjectFactory<E, T> objectFactory;
+ private final Queue<Q> recycledObjects;
+ private final T param;
+
+ protected AbstractObjectPool(IObjectFactory<E, T> objectFactory, T param) {
+ this.objectFactory = objectFactory;
+ recycledObjects = new ArrayDeque<>();
+ this.param = param;
+ }
+
+ public E getInstance() {
+ E instance = unwrap(recycledObjects.poll());
+ if (objectFactory != null && instance == null) {
+ instance = objectFactory.create(param);
+ }
+ return instance;
+ }
+
+ public void recycle(E object) {
+ if (object != null) {
+ recycledObjects.add(wrap(object));
+ }
+ }
+
+ protected abstract E unwrap(Q element);
+
+ protected abstract Q wrap(E element);
+
+ @Override
+ public String toString() {
+ return recycledObjects.toString();
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
new file mode 100644
index 0000000..a2990ec
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.parser.jackson;
+
+/**
+ * Object pool for DFS traversal mode, which allows to recycle objects
+ * as soon as it is not needed.
+ */
+public interface IObjectPool<E> {
+ E getInstance();
+
+ void recycle(E object);
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
index 8945e71..864d9a9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
@@ -18,20 +18,13 @@
*/
package org.apache.asterix.external.parser.jackson;
-import java.util.ArrayDeque;
-import java.util.Queue;
-
import org.apache.asterix.om.util.container.IObjectFactory;
/**
* Object pool for DFS traversal mode, which allows to recycle objects
* as soon as it is not needed.
*/
-public class ObjectPool<E, T> {
- private final IObjectFactory<E, T> objectFactory;
- private final Queue<E> recycledObjects;
- private final T element;
-
+public class ObjectPool<E, T> extends AbstractObjectPool<E, T, E> {
public ObjectPool() {
this(null, null);
}
@@ -40,28 +33,17 @@
this(objectFactory, null);
}
- public ObjectPool(IObjectFactory<E, T> objectFactory, T element) {
- this.objectFactory = objectFactory;
- recycledObjects = new ArrayDeque<>();
- this.element = element;
- }
-
- public E getInstance() {
- E instance = recycledObjects.poll();
- if (objectFactory != null && instance == null) {
- instance = objectFactory.create(element);
- }
- return instance;
- }
-
- public void recycle(E object) {
- if (object != null) {
- recycledObjects.add(object);
- }
+ public ObjectPool(IObjectFactory<E, T> objectFactory, T param) {
+ super(objectFactory, param);
}
@Override
- public String toString() {
- return recycledObjects.toString();
+ protected E unwrap(E wrapped) {
+ return wrapped;
+ }
+
+ @Override
+ protected E wrap(E element) {
+ return element;
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
index d0b79b1..ef9ff08 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
@@ -51,7 +51,7 @@
public class ParserContext {
private static final int SERIALIZED_FIELDNAME_MAP_MAX_SIZE = 128;
- private final ObjectPool<IARecordBuilder, ATypeTag> objectBuilderPool;
+ private final IObjectPool<IARecordBuilder> objectBuilderPool;
private final ObjectPool<IAsterixListBuilder, ATypeTag> arrayBuilderPool;
/**
@@ -61,7 +61,7 @@
* <p>
* Scalar value 5 is written 4 times in tempBuffer("d") then tempBuffer("c") ... tempBuffer("a")
*/
- private final ObjectPool<IMutableValueStorage, ATypeTag> tempBufferPool;
+ private final IObjectPool<IMutableValueStorage> tempBufferPool;
private final ObjectPool<BitSet, Void> nullBitmapPool;
private final Map<String, IMutableValueStorage> serializedFieldNames;
private final ISerializerDeserializer<AString> stringSerDe;
@@ -76,9 +76,9 @@
@SuppressWarnings("unchecked")
public ParserContext(boolean allocateModfiedUTF8Writer) {
- objectBuilderPool = new ObjectPool<>(new RecordBuilderFactory());
+ objectBuilderPool = new SoftObjectPool<>(new RecordBuilderFactory());
arrayBuilderPool = new ObjectPool<>(new ListBuilderFactory(), ATypeTag.ARRAY);
- tempBufferPool = new ObjectPool<>(new AbvsBuilderFactory());
+ tempBufferPool = new SoftObjectPool<>(new AbvsBuilderFactory());
nullBitmapPool = new ObjectPool<>();
serializedFieldNames = new LRUMap<>(SERIALIZED_FIELDNAME_MAP_MAX_SIZE);
stringSerDe = SerializerDeserializerProvider.INSTANCE.getAStringSerializerDeserializer();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
new file mode 100644
index 0000000..91655db
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.parser.jackson;
+
+import java.lang.ref.SoftReference;
+
+import org.apache.asterix.om.util.container.IObjectFactory;
+
+/**
+ * Object pool for DFS traversal mode, which allows to recycle objects
+ * as soon as it is not needed.
+ */
+public class SoftObjectPool<E, T> extends AbstractObjectPool<E, T, SoftReference<E>> {
+ public SoftObjectPool() {
+ this(null, null);
+ }
+
+ public SoftObjectPool(IObjectFactory<E, T> objectFactory) {
+ this(objectFactory, null);
+ }
+
+ public SoftObjectPool(IObjectFactory<E, T> objectFactory, T element) {
+ super(objectFactory, element);
+ }
+
+ @Override
+ protected E unwrap(SoftReference<E> wrapped) {
+ return wrapped == null ? null : wrapped.get();
+ }
+
+ @Override
+ protected SoftReference<E> wrap(E element) {
+ return new SoftReference<>(element);
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 2c29e60..8e94263 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,7 +18,10 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
@@ -77,6 +80,7 @@
import org.apache.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.LongParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import org.apache.hyracks.util.StorageUtil;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
@@ -84,8 +88,9 @@
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.ListBlobsOptions;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkException;
@@ -102,6 +107,8 @@
public class ExternalDataUtils {
private static final Map<ATypeTag, IValueParserFactory> valueParserFactoryMap = new EnumMap<>(ATypeTag.class);
+ private static final int DEFAULT_MAX_ARGUMENT_SZ = 1024 * 1024;
+ private static final int HEADER_FUDGE = 64;
static {
valueParserFactoryMap.put(ATypeTag.INTEGER, IntegerParserFactory.INSTANCE);
@@ -726,8 +733,8 @@
*/
public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
- String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey = configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME);
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
String sessionToken = configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME);
String regionId = configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME);
String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME);
@@ -735,14 +742,23 @@
S3ClientBuilder builder = S3Client.builder();
// Credentials
- AwsCredentials credentials;
- if (sessionToken != null) {
- credentials = AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken);
+ AwsCredentialsProvider credentialsProvider;
+
+ // No auth required
+ if (accessKeyId == null) {
+ credentialsProvider = AnonymousCredentialsProvider.create();
} else {
- credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey);
+ // auth required, check for temporary or permanent credentials
+ if (sessionToken != null) {
+ credentialsProvider = StaticCredentialsProvider
+ .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+ } else {
+ credentialsProvider =
+ StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+ }
}
- builder.credentialsProvider(StaticCredentialsProvider.create(credentials));
+ builder.credentialsProvider(credentialsProvider);
builder.region(Region.of(regionId));
// Validate the service endpoint if present
@@ -777,10 +793,24 @@
throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
}
+ // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ if (accessKeyId == null || secretAccessKey == null) {
+ // If one is passed, the other is required
+ if (accessKeyId != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+ ACCESS_KEY_ID_FIELD_NAME);
+ } else if (secretAccessKey != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SECRET_ACCESS_KEY_FIELD_NAME);
+ }
+ }
+
validateIncludeExclude(configuration);
// Check if the bucket is present
- S3Client s3Client = buildAwsS3Client(configuration);;
+ S3Client s3Client = buildAwsS3Client(configuration);
S3Response response;
boolean useOldApi = false;
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
@@ -970,4 +1000,20 @@
}
}
}
+
+ public static int roundUpToNearestFrameSize(int size, int framesize) {
+ return ((size / framesize) + 1) * framesize;
+ }
+
+ public static int getArgBufferSize() {
+ int maxArgSz = DEFAULT_MAX_ARGUMENT_SZ + HEADER_FUDGE;
+ String userArgSz = System.getProperty("udf.buf.size");
+ if (userArgSz != null) {
+ long parsedSize = StorageUtil.getByteValue(userArgSz) + HEADER_FUDGE;
+ if (parsedSize < Integer.MAX_VALUE && parsedSize > 0) {
+ maxArgSz = (int) parsedSize;
+ }
+ }
+ return maxArgSz;
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
deleted file mode 100644
index 5a46af7..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.util;
-
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.Warning;
-
-public class ParseUtil {
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector
- .warn(Warning.of(null, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java
new file mode 100644
index 0000000..335c383
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionFactory;
+
+public class TypeNameFactory implements IFunctionFactory {
+ @Override
+ public IExternalScalarFunction getExternalFunction() {
+ return new TypeNameFunction();
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java
new file mode 100644
index 0000000..1f03fcc
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionHelper;
+import org.apache.asterix.external.library.java.base.JString;
+
+public class TypeNameFunction implements IExternalScalarFunction {
+
+ private JString result;
+
+ @Override
+ public void deinitialize() {
+ // nothing to do here
+ }
+
+ @Override
+ public void evaluate(IFunctionHelper functionHelper) throws Exception {
+ String arg0TypeName = functionHelper.getArgument(0).getIAType().getTypeName();
+ result.setValue(arg0TypeName);
+ functionHelper.setResult(result);
+ }
+
+ @Override
+ public void initialize(IFunctionHelper functionHelper) {
+ result = (JString) functionHelper.getResultObject();
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 1c4ec64..a8d027f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -22,8 +22,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -33,19 +31,21 @@
*/
public class NodeGroup implements IMetadataEntity<NodeGroup> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster.
private final String groupName;
- private final Set<String> nodeNames;
+ private final List<String> nodeNames;
public NodeGroup(String groupName, List<String> nodeNames) {
this.groupName = groupName;
- if (nodeNames != null) {
- this.nodeNames = new TreeSet<>(nodeNames);
- } else {
- this.nodeNames = Collections.emptySet();
- }
+ this.nodeNames = nodeNames;
+ }
+
+ public static NodeGroup createOrdered(String groupName, List<String> nodeNames) {
+ List<String> sortedNodeNames = new ArrayList<>(nodeNames);
+ Collections.sort(sortedNodeNames);
+ return new NodeGroup(groupName, sortedNodeNames);
}
public String getNodeGroupName() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 6751171..5fd96a6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -69,7 +69,7 @@
IAType returnType = getType(function.getReturnType(), metadataProvider);
- IResultTypeComputer typeComputer = new ExternalTypeComputer(returnType, paramTypes);
+ IResultTypeComputer typeComputer = new ExternalTypeComputer(returnType, paramTypes, function.getNullCall());
ExternalFunctionLanguage lang = getExternalFunctionLanguage(function.getLanguage());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
index 4a000a3..99d8ea2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
@@ -33,8 +33,9 @@
public class ExternalTypeComputer extends AbstractResultTypeComputer {
- private IAType resultType;
- private List<IAType> paramPrimeTypes;
+ private final IAType resultType;
+ private final List<IAType> paramPrimeTypes;
+ private final boolean nullCall;
@Override
protected void checkArgType(FunctionIdentifier funcId, int argIndex, IAType type, SourceLocation sourceLoc)
@@ -47,14 +48,20 @@
}
}
- public ExternalTypeComputer(IAType resultPrimeType, List<IAType> paramPrimeTypes) {
+ public ExternalTypeComputer(IAType resultPrimeType, List<IAType> paramPrimeTypes, boolean nullCall) {
this.resultType = resultPrimeType.getTypeTag() == ATypeTag.ANY ? resultPrimeType
: AUnionType.createUnknownableType(resultPrimeType);
this.paramPrimeTypes = paramPrimeTypes;
+ this.nullCall = nullCall;
}
@Override
protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) {
return resultType;
}
+
+ @Override
+ protected boolean propagateNullAndMissing() {
+ return !nullCall;
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4ef349b..2e10d77 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -617,7 +617,7 @@
nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroup);
}
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, NodeGroup.createOrdered(nodeGroup, new ArrayList<>(ncNames)));
return nodeGroup;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 0eced16..a6e3087 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -521,4 +521,8 @@
public IFileSplitProvider getSecondaryFileSplitProvider() {
return secondaryFileSplitProvider;
}
+
+ public AlgebricksPartitionConstraint getSecondaryPartitionConstraint() {
+ return secondaryPartitionConstraint;
+ }
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index aac01f9..a2e235e 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1592,8 +1592,6 @@
public static final FunctionIdentifier DECODE_DATAVERSE_NAME =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "decode-dataverse-name", 1);
- public static final FunctionIdentifier DECODE_DATAVERSE_DISPLAY_NAME =
- new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "decode-dataverse-display-name", 1);
static {
// first, take care of Algebricks builtin functions
@@ -2366,7 +2364,6 @@
addPrivateFunction(META_KEY, AnyTypeComputer.INSTANCE, false);
addFunction(DECODE_DATAVERSE_NAME, OrderedListOfAStringTypeComputer.INSTANCE_NULLABLE, true);
- addFunction(DECODE_DATAVERSE_DISPLAY_NAME, AStringTypeComputer.INSTANCE_NULLABLE, true);
addPrivateFunction(COLLECTION_TO_SEQUENCE, CollectionToSequenceTypeComputer.INSTANCE, true);
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
index 76ba6a2..bdac9e9 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
@@ -45,7 +45,7 @@
/** the type of all types */
public static final BuiltinType ALL_TYPE = new BuiltinType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -6449893468153263063L;
@Override
public ATypeTag getTypeTag() {
@@ -83,7 +83,7 @@
public static final BuiltinType AINT8 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -3548486253352636441L;
@Override
public String getDisplayName() {
@@ -116,7 +116,7 @@
public static final BuiltinType AINT16 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 7372637344359144146L;
@Override
public String getDisplayName() {
@@ -149,7 +149,7 @@
public static final BuiltinType AINT32 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -2019098705812691759L;
@Override
public String getDisplayName() {
@@ -182,7 +182,7 @@
public static final BuiltinType AINT64 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1512324312146380842L;
@Override
public ATypeTag getTypeTag() {
@@ -215,7 +215,7 @@
public static final BuiltinType ABINARY = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 6267661034873713699L;
@Override
public String getDisplayName() {
@@ -248,7 +248,7 @@
public static final BuiltinType AFLOAT = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 3677347396583352706L;
@Override
public String getDisplayName() {
@@ -281,7 +281,7 @@
public static final BuiltinType ADOUBLE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -4541634684505456336L;
@Override
public String getDisplayName() {
@@ -314,7 +314,7 @@
public static final BuiltinType ASTRING = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -7772076532516609205L;
@Override
public String getDisplayName() {
@@ -347,7 +347,7 @@
public static final BuiltinType AMISSING = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 7776297133205477953L;
@Override
public String getDisplayName() {
@@ -380,7 +380,7 @@
public static final BuiltinType ANULL = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -8499117420923359476L;
@Override
public String getDisplayName() {
@@ -413,7 +413,7 @@
public static final BuiltinType ABOOLEAN = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -3974299905945243639L;
@Override
public String getDisplayName() {
@@ -446,7 +446,7 @@
public static final BuiltinType ATIME = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 4868038022671454583L;
@Override
public String getDisplayName() {
@@ -479,7 +479,7 @@
public static final BuiltinType ADATE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 4037452728204123123L;
@Override
public String getDisplayName() {
@@ -512,7 +512,7 @@
public static final BuiltinType ADATETIME = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 8133030540756758494L;
@Override
public String getDisplayName() {
@@ -545,7 +545,7 @@
public static final BuiltinType ADURATION = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2724423960287103412L;
@Override
public String getDisplayName() {
@@ -578,7 +578,7 @@
public static final BuiltinType AYEARMONTHDURATION = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -5424462543021800169L;
@Override
public String getDisplayName() {
@@ -611,7 +611,7 @@
public static final BuiltinType ADAYTIMEDURATION = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1605907151536226966L;
@Override
public String getDisplayName() {
@@ -644,7 +644,7 @@
public static final BuiltinType AINTERVAL = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 6503802193278692294L;
@Override
public String getDisplayName() {
@@ -674,7 +674,7 @@
public static final BuiltinType APOINT = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -8095235249315628525L;
@Override
public ATypeTag getTypeTag() {
@@ -707,7 +707,7 @@
public static final BuiltinType APOINT3D = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 8511348838989487264L;
@Override
public ATypeTag getTypeTag() {
@@ -740,7 +740,7 @@
public static final BuiltinType ALINE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -5081002479268381298L;
@Override
public String getDisplayName() {
@@ -773,7 +773,7 @@
public static final BuiltinType APOLYGON = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1150256493732890410L;
@Override
public String getDisplayName() {
@@ -806,7 +806,7 @@
public static final BuiltinType AGEOMETRY = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1561894254022047509L;
@Override
public String getDisplayName() {
@@ -838,7 +838,7 @@
public static final BuiltinType ACIRCLE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2609452848446856772L;
@Override
public String getDisplayName() {
@@ -871,7 +871,7 @@
public static final BuiltinType ARECTANGLE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 3751474563396177041L;
@Override
public String getDisplayName() {
@@ -904,7 +904,7 @@
public static final IAType ABITARRAY = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 9071500011277554477L;
@Override
public ATypeTag getTypeTag() {
@@ -936,7 +936,7 @@
};
public static final BuiltinType AUUID = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -8403791895909921192L;
@Override
public ATypeTag getTypeTag() {
@@ -969,7 +969,7 @@
public static final BuiltinType ANY = new BuiltinType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -4041457851203593309L;
@Override
public ATypeTag getTypeTag() {
@@ -1007,7 +1007,7 @@
public static final BuiltinType SHORTWITHOUTTYPEINFO = new BuiltinType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1769780932699820665L;
@Override
public ATypeTag getTypeTag() {
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index 97dd049..7f6439c 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -60,30 +60,32 @@
public static void sendFile(FileChannel fileChannel, ISocketChannel socketChannel) throws IOException {
long pos = 0;
- long fileSize = fileChannel.size();
- long remainingBytes = fileSize;
- long transferredBytes = 0;
-
- while ((transferredBytes += fileChannel.transferTo(pos, remainingBytes, socketChannel)) < fileSize) {
- pos += transferredBytes;
- remainingBytes -= transferredBytes;
+ long remainingBytes = fileChannel.size();
+ try {
+ while (remainingBytes > 0) {
+ long sentBytes = fileChannel.transferTo(pos, remainingBytes, socketChannel);
+ pos += sentBytes;
+ remainingBytes -= sentBytes;
+ }
+ socketChannel.getSocketChannel().socket().getOutputStream().flush();
+ } catch (Exception e) {
+ LOGGER.info("failed to send file; file size {}, pos {}, remainingBytes {}", fileChannel.size(), pos,
+ remainingBytes);
}
- socketChannel.getSocketChannel().socket().getOutputStream().flush();
}
public static void downloadFile(FileChannel fileChannel, ISocketChannel socketChannel) throws IOException {
+ long remainingBytes = fileChannel.size();
long pos = 0;
- long fileSize = fileChannel.size();
- long count = fileSize;
- long numTransferred = 0;
try {
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ while (remainingBytes > 0) {
+ long readBytes = fileChannel.transferFrom(socketChannel, pos, remainingBytes);
+ pos += readBytes;
+ remainingBytes -= readBytes;
}
} catch (Exception e) {
- LOGGER.info("failed to download file; file size {}, pos {}, count {}, numTransferred {}", fileSize, pos,
- count, numTransferred);
+ LOGGER.info("failed to download file; file size {}, pos {}, remainingBytes {}", fileChannel.size(), pos,
+ remainingBytes);
throw e;
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DecodeDataverseDisplayNameDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DecodeDataverseDisplayNameDescriptor.java
deleted file mode 100644
index fd78b72..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DecodeDataverseDisplayNameDescriptor.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.IOException;
-
-import org.apache.asterix.common.annotations.MissingNullInOutFunction;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-
-@MissingNullInOutFunction
-public class DecodeDataverseDisplayNameDescriptor extends AbstractScalarFunctionDynamicDescriptor {
- private static final long serialVersionUID = 1L;
-
- public static final IFunctionDescriptorFactory FACTORY = DecodeDataverseDisplayNameDescriptor::new;
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
- return new AbstractUnaryStringStringEval(ctx, args[0], getIdentifier(), sourceLoc) {
-
- private final StringBuilder sb = new StringBuilder();
-
- @Override
- void process(UTF8StringPointable inputString, IPointable resultPointable) throws IOException {
- String dataverseCanonicalName = inputString.toString();
-
- sb.setLength(0);
- try {
- DataverseName.getDisplayFormFromCanonicalForm(dataverseCanonicalName, sb);
- } catch (AsterixException e) {
- return; // writeResult() will emit NULL
- }
-
- resultBuilder.reset(resultArray, inputString.getUTF8Length());
- resultBuilder.appendString(sb);
- resultBuilder.finish();
- }
-
- @Override
- void writeResult(IPointable resultPointable) throws IOException {
- if (sb.length() == 0) {
- PointableHelper.setNull(resultPointable);
- } else {
- super.writeResult(resultPointable);
- }
- }
- };
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.DECODE_DATAVERSE_DISPLAY_NAME;
- }
-}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index 17159d3..18d55aa 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@ -331,7 +331,6 @@
import org.apache.asterix.runtime.evaluators.functions.CreateQueryUIDDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateRectangleDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateUUIDDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.DecodeDataverseDisplayNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DecodeDataverseNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DeepEqualityDescriptor;
import org.apache.asterix.runtime.evaluators.functions.FullTextContainsFunctionDescriptor;
@@ -1215,7 +1214,6 @@
// Other functions
fc.add(DecodeDataverseNameDescriptor.FACTORY);
- fc.add(DecodeDataverseDisplayNameDescriptor.FACTORY);
fc.add(RandomWithSeedDescriptor.FACTORY);
ServiceLoader.load(IFunctionRegistrant.class).iterator().forEachRemaining(c -> c.register(fc));
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index bf7ca34..98a97b0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -301,8 +301,6 @@
}
}
clusterActiveLocations.removeAll(pendingRemoval);
- // for operators attempting to access storage, order the nodes list similar to a nodegroup
- Collections.sort(clusterActiveLocations);
clusterPartitionConstraint =
new AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new String[] {}));
}
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index 4c0fc3f..ce943ad 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -171,8 +171,8 @@
<url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url>
</override>
<override>
- <gav>io.netty:netty-all:4.1.59.Final</gav>
- <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.59.Final/NOTICE.txt</noticeUrl>
+ <gav>io.netty:netty-all:4.1.63.Final</gav>
+ <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.63.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>org.reactivestreams:reactive-streams:1.0.2</gav>
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
index 93cda83..a82b302 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
@@ -41,7 +41,7 @@
ResourceGroup get(int dId, int entityHashValue) {
// TODO ensure good properties of hash function
- return table[Math.abs(dId ^ entityHashValue) % size];
+ return table[Math.abs((dId ^ entityHashValue) % size)];
}
ResourceGroup get(int i) {
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 6067ef9..a930643 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -84,7 +84,7 @@
<hyracks.version>0.3.6-SNAPSHOT</hyracks.version>
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.13.3</log4j.version>
+ <log4j.version>2.14.1</log4j.version>
<awsjavasdk.version>2.10.83</awsjavasdk.version>
<azurejavasdk.version>12.6.0</azurejavasdk.version>
<parquet.version>1.8.2</parquet.version>
@@ -1385,7 +1385,7 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.14</version>
+ <version>1.15</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index 7f3b7ec..c352fe5 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -160,9 +160,9 @@
<artifactId>netty-all</artifactId>
<properties>
<!-- netty is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
- <license.ignoreMissingEmbeddedLicense>4.1.59.Final</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>4.1.59.Final</license.ignoreMissingEmbeddedNotice>
- <license.ignoreNoticeOverride>4.1.59.Final</license.ignoreNoticeOverride>
+ <license.ignoreMissingEmbeddedLicense>4.1.63.Final</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>4.1.63.Final</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreNoticeOverride>4.1.63.Final</license.ignoreNoticeOverride>
</properties>
</project>
</supplement>
@@ -229,7 +229,7 @@
<artifactId>jackson-annotations</artifactId>
<properties>
<!-- jackson-annotations does not provide an embedded NOTICE file -->
- <license.ignoreMissingEmbeddedNotice>2.8.4,2.9.7,2.9.10,2.10.0,2.10.3,2.12.1</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreMissingEmbeddedNotice>2.12.3</license.ignoreMissingEmbeddedNotice>
</properties>
</project>
</supplement>
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt
similarity index 100%
rename from asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt
rename to asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
index a243996..ed5299c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
@@ -411,8 +411,13 @@
@Override
public Void visitSubplanOperator(SubplanOperator op, Void indent) throws AlgebricksException {
- if (!op.getNestedPlans().isEmpty()) {
- writeNestedPlans(op, indent);
+ try {
+ if (!op.getNestedPlans().isEmpty()) {
+ jsonGenerator.writeStringField(OPERATOR_FIELD, "subplan");
+ writeNestedPlans(op, indent);
+ }
+ } catch (IOException e) {
+ throw AlgebricksException.create(ErrorCode.ERROR_PRINTING_PLAN, e, String.valueOf(e));
}
return null;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
index 5386193..3760be5 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
@@ -36,7 +36,6 @@
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
@@ -69,26 +68,33 @@
public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
private final VariableSubstitutionVisitor substVisitor = new VariableSubstitutionVisitor();
- private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap =
- new HashMap<LogicalVariable, List<LogicalVariable>>();
+
+ private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap = new HashMap<>();
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
if (context.checkIfInDontApplySet(this, opRef.getValue())) {
return false;
}
- boolean modified = removeRedundantVariables(opRef, context);
- if (modified) {
- context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
- }
- return modified;
+ clear();
+ return removeRedundantVariables(opRef, true, context);
+ }
+
+ private void clear() {
+ equivalentVarsMap.clear();
}
private void updateEquivalenceClassMap(LogicalVariable lhs, LogicalVariable rhs) {
List<LogicalVariable> equivalentVars = equivalentVarsMap.get(rhs);
if (equivalentVars == null) {
- equivalentVars = new ArrayList<LogicalVariable>();
+ equivalentVars = new ArrayList<>();
// The first element in the list is the bottom-most representative which will replace all equivalent vars.
equivalentVars.add(rhs);
equivalentVars.add(lhs);
@@ -97,12 +103,32 @@
equivalentVarsMap.put(lhs, equivalentVars);
}
- private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
+ private LogicalVariable findEquivalentRepresentativeVar(LogicalVariable var) {
+ List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
+ if (equivalentVars == null) {
+ return null;
+ }
+ LogicalVariable representativeVar = equivalentVars.get(0);
+ return var.equals(representativeVar) ? null : representativeVar;
+ }
+
+ private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, boolean first,
+ IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (!first) {
+ context.addToDontApplySet(this, op);
+ }
+
LogicalOperatorTag opTag = op.getOperatorTag();
boolean modified = false;
+ // Recurse into children.
+ for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+ if (removeRedundantVariables(inputOpRef, false, context)) {
+ modified = true;
+ }
+ }
+
// Update equivalence class map.
if (opTag == LogicalOperatorTag.ASSIGN) {
AssignOperator assignOp = (AssignOperator) op;
@@ -142,7 +168,7 @@
AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
- if (removeRedundantVariables(rootRef, context)) {
+ if (removeRedundantVariables(rootRef, false, context)) {
modified = true;
}
}
@@ -158,14 +184,8 @@
if (modified) {
context.computeAndSetTypeEnvironmentForOperator(op);
- context.addToDontApplySet(this, op);
}
- // Clears the equivalent variable map if the current operator is the root operator
- // in the query plan.
- if (opTag == LogicalOperatorTag.DISTRIBUTE_RESULT || opTag == LogicalOperatorTag.SINK) {
- equivalentVarsMap.clear();
- }
return modified;
}
@@ -227,38 +247,34 @@
* We cannot use the VariableSubstitutionVisitor here because the project ops
* maintain their variables as a list and not as expressions.
*/
- private boolean replaceProjectVars(ProjectOperator op) throws AlgebricksException {
+ private boolean replaceProjectVars(ProjectOperator op) {
List<LogicalVariable> vars = op.getVariables();
int size = vars.size();
boolean modified = false;
for (int i = 0; i < size; i++) {
LogicalVariable var = vars.get(i);
- List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
- if (equivalentVars == null) {
- continue;
- }
- // Replace with equivalence class representative.
- LogicalVariable representative = equivalentVars.get(0);
- if (representative != var) {
- vars.set(i, equivalentVars.get(0));
+ LogicalVariable representativeVar = findEquivalentRepresentativeVar(var);
+ if (representativeVar != null) {
+ // Replace with equivalence class representative.
+ vars.set(i, representativeVar);
modified = true;
}
}
return modified;
}
- private boolean replaceUnionAllVars(UnionAllOperator op) throws AlgebricksException {
+ private boolean replaceUnionAllVars(UnionAllOperator op) {
boolean modified = false;
for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : op.getVariableMappings()) {
- List<LogicalVariable> firstEquivalentVars = equivalentVarsMap.get(varMapping.first);
- List<LogicalVariable> secondEquivalentVars = equivalentVarsMap.get(varMapping.second);
// Replace variables with their representative.
- if (firstEquivalentVars != null) {
- varMapping.first = firstEquivalentVars.get(0);
+ LogicalVariable firstRepresentativeVar = findEquivalentRepresentativeVar(varMapping.first);
+ if (firstRepresentativeVar != null) {
+ varMapping.first = firstRepresentativeVar;
modified = true;
}
- if (secondEquivalentVars != null) {
- varMapping.second = secondEquivalentVars.get(0);
+ LogicalVariable secondRepresentativeVar = findEquivalentRepresentativeVar(varMapping.second);
+ if (secondRepresentativeVar != null) {
+ varMapping.second = secondRepresentativeVar;
modified = true;
}
}
@@ -269,17 +285,13 @@
@Override
public boolean transform(Mutable<ILogicalExpression> exprRef) {
ILogicalExpression e = exprRef.getValue();
- switch (((AbstractLogicalExpression) e).getExpressionTag()) {
+ switch (e.getExpressionTag()) {
case VARIABLE: {
// Replace variable references with their equivalent representative in the equivalence class map.
VariableReferenceExpression varRefExpr = (VariableReferenceExpression) e;
LogicalVariable var = varRefExpr.getVariableReference();
- List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
- if (equivalentVars == null) {
- return false;
- }
- LogicalVariable representative = equivalentVars.get(0);
- if (representative != var) {
+ LogicalVariable representative = findEquivalentRepresentativeVar(var);
+ if (representative != null) {
varRefExpr.setVariable(representative);
return true;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java
index ddcfd78..55eaf68 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java
@@ -54,6 +54,11 @@
}
@Override
+ public String getDisplayName() {
+ return "SA";
+ }
+
+ @Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
throws HyracksDataException {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
index 79687ab..e43d72a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
@@ -55,7 +55,7 @@
* connected activities in a single thread.
*/
public class SuperActivityOperatorNodePushable implements IOperatorNodePushable {
- private static final String CLASS_ABBREVIATION = "SA";
+ private static final String CLASS_ABBREVIATION = "SAO";
private final Map<ActivityId, IOperatorNodePushable> operatorNodePushables = new HashMap<>();
private final List<IOperatorNodePushable> operatorNodePushablesBFSOrder = new ArrayList<>();
private final Map<ActivityId, IActivity> startActivities;
@@ -222,9 +222,8 @@
tasks.add(ctx.getExecutorService().submit(() -> {
startSemaphore.release();
try {
- Thread.currentThread()
- .setName(Thread.currentThread().getName() + ":" + ctx.getJobletContext().getJobId()
- + ":" + ctx.getTaskAttemptId() + ":" + CLASS_ABBREVIATION);
+ Thread.currentThread().setName(CLASS_ABBREVIATION + ":" + ctx.getJobletContext().getJobId()
+ + ":" + ctx.getTaskAttemptId());
action.run(op);
} catch (Throwable th) { // NOSONAR: Must catch all causes of failure
failures.offer(th);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
index 6479c8d..70b13fa 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
@@ -102,12 +102,15 @@
try (Formatter fmt = new Formatter()) {
if (!NONE.equals(component)) {
fmt.format("%1$s%2$04d: ", component, errorCode);
+
+ // if the message is already formatted, just return it
+ if (message.startsWith(fmt.toString())) {
+ return message;
+ }
}
- // if the message is already formatted, just return it
- if (!fmt.toString().isEmpty() && message.startsWith(fmt.toString())) {
- return message;
+ if (message != null) {
+ fmt.format(message, (Object[]) params);
}
- fmt.format(message == null ? "null" : message, (Object[]) params);
if (sourceLoc != null) {
fmt.out().append(" (in line ").append(String.valueOf(sourceLoc.getLine())).append(", at column ")
.append(String.valueOf(sourceLoc.getColumn())).append(')');
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingSupplier.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingSupplier.java
new file mode 100644
index 0000000..c6c3922
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingSupplier.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.util;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+@FunctionalInterface
+public interface HyracksThrowingSupplier<T> {
+ T get() throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
index 0b1c5a6..7d04cf2 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
@@ -280,6 +280,9 @@
long delayMs = delay.calculate(attempt);
if (!policy.retry(th) || span.elapsed() || span.remaining(TimeUnit.MILLISECONDS) < delayMs) {
onFailure.attemptFailed(action, attempt, true, span, failure);
+ if (th instanceof Error) {
+ throw (Error) th;
+ }
throw HyracksDataException.create(failure);
} else {
onFailure.attemptFailed(action, attempt, false, span, failure);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/ParseUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/ParseUtil.java
new file mode 100644
index 0000000..63fec09
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/ParseUtil.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.util.StringJoiner;
+
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.Warning;
+
+public class ParseUtil {
+
+ private ParseUtil() {
+ }
+
+ public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
+ String warnMessage) {
+ warningCollector.warn(Warning.of(null, ErrorCode.PARSING_ERROR,
+ asLocationDetailString(dataSourceName, lineNum, fieldNum), warnMessage));
+ }
+
+ public static String asLocationDetailString(String dataSource, long lineNum, Object fieldIdentifier) {
+ StringJoiner details = new StringJoiner(" ");
+ details.setEmptyValue("N/A");
+ if (dataSource != null && !dataSource.isEmpty()) {
+ details.add(dataSource);
+ }
+ if (lineNum >= 0) {
+ details.add("line " + lineNum);
+ }
+ if (fieldIdentifier instanceof Number) {
+ details.add("field " + fieldIdentifier);
+ } else if (fieldIdentifier instanceof String && !((String) fieldIdentifier).isEmpty()) {
+ details.add("field '" + fieldIdentifier + "'");
+ }
+ return "at " + details;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 9f04fb2..4d9c60b 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -141,7 +141,7 @@
121 = A numeric type promotion error has occurred: %1$s
122 = Encountered an error while printing the plan: %1$s
123 = Insufficient memory is provided for the join operators, please increase the join memory budget.
-124 = Parsing error at %1$s line %2$s field %3$s: %4$s
+124 = Parsing error %s: %s
125 = Invalid inverted list type traits: %1$s
126 = Illegal state. %1$s
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
index 7e1ca61..d782ac5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
@@ -84,7 +84,7 @@
}
activeRunMap = new HashMap<>();
runMapArchive = new LinkedHashMap<JobId, JobRun>() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1406441385508773629L;
@Override
protected boolean removeEldestEntry(Map.Entry<JobId, JobRun> eldest) {
@@ -92,7 +92,7 @@
}
};
runMapHistory = new LinkedHashMap<JobId, List<Exception>>() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 7572062687032652986L;
/** history size + 1 is for the case when history size = 0 */
private final int allowedSize = 100 * (ccConfig.getJobHistorySize() + 1);
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
index 936d63e..ffc87cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
@@ -23,9 +23,8 @@
import java.util.Arrays;
import java.util.function.Supplier;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.util.ParseUtil;
public class FieldCursorForDelimitedDataParser {
@@ -448,6 +447,6 @@
}
private void warn(String message) {
- warnings.warn(Warning.of(null, ErrorCode.PARSING_ERROR, dataSourceName.get(), lineCount, fieldCount, message));
+ ParseUtil.warn(warnings, dataSourceName.get(), lineCount, fieldCount, message);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
index 514a7dd..ed567f5 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
@@ -53,20 +53,20 @@
protected final String[] paths;
protected final ConcurrentMap<String, Object> ctx;
- protected final int[] trims;
+ protected final int[] servletPathLengths;
public AbstractServlet(ConcurrentMap<String, Object> ctx, String... paths) {
this.paths = paths;
this.ctx = ctx;
- trims = new int[paths.length];
+ servletPathLengths = new int[paths.length];
for (int i = 0; i < paths.length; i++) {
String path = paths[i];
if (path.endsWith("/*")) {
- trims[i] = path.indexOf("/*");
+ servletPathLengths[i] = path.indexOf("/*");
} else if (path.endsWith("/")) {
- trims[i] = path.length() - 1;
+ servletPathLengths[i] = path.length() - 1;
} else {
- trims[i] = path.length();
+ servletPathLengths[i] = path.length();
}
}
}
@@ -175,26 +175,27 @@
public String localPath(IServletRequest request) {
final String uri = request.getHttpRequest().uri();
int queryStart = uri.indexOf('?');
- return queryStart == -1 ? uri.substring(trim(uri)) : uri.substring(trim(uri), queryStart);
+ return queryStart == -1 ? uri.substring(servletLength(uri)) : uri.substring(servletLength(uri), queryStart);
}
public String servletPath(IServletRequest request) {
final String uri = request.getHttpRequest().uri();
- return uri.substring(0, trim(uri));
+ return uri.substring(0, servletLength(uri));
}
- protected int trim(final String uri) {
+ protected int servletLength(final String uri) {
int trim = -1;
if (paths.length > 1) {
for (int i = 0; i < paths.length; i++) {
- String path = paths[i].indexOf('*') >= 0 ? paths[i].substring(0, paths[i].indexOf('*')) : paths[i];
+ int wildCardIdx = paths[i].indexOf("/*");
+ String path = wildCardIdx >= 0 ? paths[i].substring(0, wildCardIdx) : paths[i];
if (uri.indexOf(path) == 0) {
- trim = trims[i];
+ trim = servletPathLengths[i];
break;
}
}
} else {
- trim = trims[0];
+ trim = servletPathLengths[0];
}
return trim;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
index 77d3493..a7ace7a 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
@@ -18,8 +18,6 @@
*/
package org.apache.hyracks.http.server;
-import static org.apache.hyracks.http.server.utils.HttpUtil.X_FORWARDED_PROTO;
-
import java.io.IOException;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
@@ -143,9 +141,7 @@
private void submit(ChannelHandlerContext ctx, IServlet servlet, FullHttpRequest request) throws IOException {
IServletRequest servletRequest;
try {
- HttpScheme scheme =
- server.getScheme() == HttpScheme.HTTPS || "https".equals(request.headers().get(X_FORWARDED_PROTO))
- ? HttpScheme.HTTPS : HttpScheme.HTTP;
+ HttpScheme scheme = HttpUtil.getScheme(server, request);
servletRequest = createServletRequest(ctx, request, scheme);
} catch (IllegalArgumentException e) {
LOGGER.log(Level.WARN, "Failure Decoding Request", e);
@@ -166,7 +162,7 @@
}
}
- protected void handleServletNotFound(ChannelHandlerContext ctx, FullHttpRequest request) {
+ protected void handleServletNotFound(ChannelHandlerContext ctx, FullHttpRequest request) throws IOException {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("No servlet for " + request.uri());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 01baa69..835cd54 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -43,7 +43,9 @@
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.BaseRequest;
import org.apache.hyracks.http.server.FormUrlEncodedRequest;
+import org.apache.hyracks.http.server.HttpServer;
import org.apache.hyracks.util.ThrowingConsumer;
+import org.apache.hyracks.util.ThrowingFunction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -190,14 +192,23 @@
return i < 0 ? uri : uri.substring(0, i);
}
- public static void handleStreamInterruptibly(CloseableHttpResponse response,
- ThrowingConsumer<Reader> streamProcessor, ExecutorService executor, Supplier<String> taskDescription)
+ public static void consumeStreamInterruptibly(CloseableHttpResponse response,
+ ThrowingConsumer<Reader> streamProcessor, ExecutorService executor, Supplier<String> descriptionSupplier)
+ throws InterruptedException, ExecutionException, IOException {
+ processStreamInterruptibly(response, ThrowingConsumer.asFunction(streamProcessor), executor,
+ descriptionSupplier);
+ }
+
+ public static <T> T processStreamInterruptibly(CloseableHttpResponse response,
+ ThrowingFunction<Reader, T> streamProcessor, ExecutorService executor, Supplier<String> descriptionSupplier)
throws IOException, InterruptedException, ExecutionException {
// we have to consume the stream in a separate thread, as it not stop on interrupt; we need to
// instead close the connection to achieve the interrupt
- Future<Void> readFuture = executor.submit(() -> {
+ String description = descriptionSupplier.get();
+ Future<T> readFuture = executor.submit(() -> {
+ Thread.currentThread().setName(description);
InputStreamReader reader = new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8);
- streamProcessor.process(new Reader() {
+ return streamProcessor.process(new Reader() {
@Override
public int read(char[] cbuf, int off, int len) throws IOException {
return reader.read(cbuf, off, len);
@@ -210,24 +221,28 @@
LOGGER.debug("ignoring close on {}", reader);
}
});
- return null;
});
try {
- readFuture.get();
+ return readFuture.get();
} catch (InterruptedException ex) { // NOSONAR -- interrupt or rethrow
response.close();
try {
readFuture.get(1, TimeUnit.SECONDS);
} catch (TimeoutException te) {
- LOGGER.warn("{} did not exit on stream close due to interrupt after 1s", taskDescription);
+ LOGGER.warn("{} did not exit on stream close due to interrupt after 1s", description);
readFuture.cancel(true);
} catch (ExecutionException ee) {
- LOGGER.debug("ignoring exception awaiting aborted {} shutdown", taskDescription, ee);
+ LOGGER.debug("ignoring exception awaiting aborted {} shutdown", description, ee);
}
throw ex;
}
}
+ public static HttpScheme getScheme(HttpServer server, FullHttpRequest request) {
+ return server.getScheme() == HttpScheme.HTTPS || "https".equals(request.headers().get(X_FORWARDED_PROTO))
+ ? HttpScheme.HTTPS : HttpScheme.HTTP;
+ }
+
public static class ContentType {
public static final String ADM = "adm";
public static final String JSON = "json";
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index abd9fda..f4c4183 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -103,9 +103,11 @@
public static synchronized void halt(int status, Level logLevel) {
try {
+ boolean interrupted = Thread.interrupted();
+ LOGGER.log(logLevel, "JVM halting with status {} (halting thread {}, interrupted {})", status,
+ Thread.currentThread(), interrupted);
Future<?> future = haltThreadDumpExecutor.submit(() -> {
- LOGGER.log(logLevel, "JVM halting with status {}; thread dump at halt: {}", status,
- ThreadDumpUtil.takeDumpString());
+ LOGGER.log(logLevel, "Thread dump at halt: {}", ThreadDumpUtil.takeDumpString());
// try to give time for the log to be emitted...
LogManager.shutdown();
});
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java
index a9ebb50..9adac04 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java
@@ -18,8 +18,6 @@
*/
package org.apache.hyracks.util;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
@@ -28,7 +26,7 @@
public class MaintainedThreadNameExecutorService extends ThreadPoolExecutor {
- private final Map<Thread, String> threadNames = new ConcurrentHashMap<>();
+ private static final ThreadLocal<String> savedName = new ThreadLocal<>();
private MaintainedThreadNameExecutorService(ThreadFactory threadFactory) {
super(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), threadFactory);
@@ -40,17 +38,13 @@
@Override
protected void beforeExecute(Thread t, Runnable r) {
- threadNames.put(t, t.getName());
+ savedName.set(t.getName());
super.beforeExecute(t, r);
}
@Override
protected void afterExecute(Runnable r, Throwable t) {
super.afterExecute(r, t);
- final Thread thread = Thread.currentThread();
- final String originalThreadName = threadNames.remove(thread);
- if (originalThreadName != null) {
- thread.setName(originalThreadName);
- }
+ Thread.currentThread().setName(savedName.get());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
index 958310d..4f0c3a8 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
@@ -110,6 +110,28 @@
return address != null ? toHostPort(address.getHostString(), address.getPort()) : null;
}
+ public static String toHostPort(HttpHost httpHost) {
+ if (httpHost == null) {
+ return null;
+ }
+ int port = httpHost.getPort();
+ if (port == -1) {
+ port = "https".equalsIgnoreCase(httpHost.getSchemeName()) ? 443 : 80;
+ }
+ return toHostPort(httpHost.getHostName(), port);
+ }
+
+ public static String toHostPort(HttpHost httpHost, int defaultPort) {
+ if (httpHost == null) {
+ return null;
+ }
+ int port = httpHost.getPort();
+ if (port == -1) {
+ port = defaultPort;
+ }
+ return toHostPort(httpHost.getHostName(), port);
+ }
+
public static InetSocketAddress parseInetSocketAddress(String hostPortString) {
int lastColon = hostPortString.lastIndexOf(':');
String host = decodeIPv6LiteralHost(lastColon < 0 ? hostPortString : hostPortString.substring(0, lastColon));
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
index e4969f0..0456dc7 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
@@ -20,10 +20,13 @@
import java.util.HashMap;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
public class StorageUtil {
public static final int BASE = 1024;
+ private static final Pattern PATTERN = Pattern.compile("^(-?[.0-9]+)([A-Z]{0,2})$");
public enum StorageUnit {
BYTE("B", "b", 1),
@@ -96,43 +99,27 @@
* @throws IllegalArgumentException
*/
public static double getSizeInBytes(String s) {
- String sSpaceRemoved = s.replaceAll(" ", "");
- String sUpper = sSpaceRemoved.toUpperCase();
-
- // Default type
- StorageUtil.StorageUnit unitType;
-
- // If the length is 1, it should only contain a digit number.
- if (sUpper.length() == 1) {
- if (Character.isDigit(sUpper.charAt(0))) {
- unitType = StorageUnit.BYTE;
- } else {
- throw invalidFormatException(s);
- }
- } else if (sUpper.length() > 1) {
- String checkStr = sUpper.substring(sUpper.length() - 2);
- unitType = StorageUnit.lookupBySuffix(checkStr);
-
- if (unitType == null) {
- // The last suffix should be at least "B" or a digit to be qualified as byte unit string.
- char lastChar = sUpper.charAt(sUpper.length() - 1);
- if (sUpper.substring(sUpper.length() - 1).equals(StorageUnit.BYTE.toString())
- || Character.isDigit(lastChar)) {
- unitType = StorageUnit.BYTE;
- } else {
- throw invalidFormatException(s);
- }
- }
- } else {
- // String length is zero. We can't parse this string.
+ String valueAndUnit = s.replace(" ", "").toUpperCase();
+ Matcher matcher = PATTERN.matcher(valueAndUnit);
+ if (!matcher.find()) {
throw invalidFormatException(s);
}
- // Strip all unit suffixes such as KB, MB ...
- String sFinalVal = sUpper.replaceAll("[^-\\.0123456789]", "");
+ String value = matcher.group(1);
+ String unit = matcher.group(2);
- // Return the bytes.
- return unitType.toBytes(Double.parseDouble(sFinalVal));
+ // Default to bytes or find provided unit
+ StorageUnit unitType = !unit.isEmpty() ? StorageUnit.lookupBySuffix(unit) : StorageUnit.BYTE;
+ if (unitType == null) {
+ throw invalidFormatException(s);
+ }
+
+ try {
+ // Return the bytes.
+ return unitType.toBytes(Double.parseDouble(value));
+ } catch (NumberFormatException ex) {
+ throw invalidFormatException(s);
+ }
}
private static IllegalArgumentException invalidFormatException(String s) {
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java
index d18b41b..f60efbc 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java
@@ -18,7 +18,26 @@
*/
package org.apache.hyracks.util;
+import java.util.function.Function;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
@FunctionalInterface
public interface ThrowingFunction<I, R> {
R process(I input) throws Exception;
+
+ @SuppressWarnings("Duplicates")
+ static <I, R> Function<I, R> asUnchecked(ThrowingFunction<I, R> consumer) {
+ return input -> {
+ try {
+ return consumer.process(input);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new UncheckedExecutionException(e);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e);
+ }
+ };
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
index e83aedb..3eb8687 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -25,6 +25,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.io.UTFDataFormatException;
+import java.lang.ref.SoftReference;
import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
@@ -694,10 +695,12 @@
if (writer == null) {
tempBytes = new byte[utflen + 5];
} else {
- if (writer.tempBytes == null || writer.tempBytes.length < utflen + 5) {
- writer.tempBytes = new byte[utflen + 5];
+ byte[] writerTempBytes = writer.tempBytesRef != null ? writer.tempBytesRef.get() : null;
+ if (writerTempBytes == null || writerTempBytes.length < utflen + 5) {
+ writerTempBytes = new byte[utflen + 5];
+ writer.tempBytesRef = new SoftReference<>(writerTempBytes);
}
- tempBytes = writer.tempBytes;
+ tempBytes = writerTempBytes;
}
return tempBytes;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
index 563f865..a0cc7d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
@@ -21,11 +21,12 @@
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
+import java.lang.ref.SoftReference;
public class UTF8StringWriter implements Serializable {
private static final long serialVersionUID = 1L;
- transient byte[] tempBytes;
+ transient SoftReference<byte[]> tempBytesRef;
public final void writeUTF8(CharSequence str, DataOutput out) throws IOException {
UTF8StringUtil.writeUTF8(str, out, this);
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java
index c5d42c5..9300808 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java
@@ -18,6 +18,9 @@
*/
package org.apache.hyracks.util;
+import java.net.URI;
+
+import org.apache.http.client.utils.URIUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Assert;
@@ -38,5 +41,13 @@
Assert.assertEquals("localhost.localdomain.local:1234",
NetworkUtil.defaultPort("localhost.localdomain.local:1234", 9999));
+ Assert.assertEquals("[::1]:1234",
+ NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("http://[::1]:1234"))));
+ Assert.assertEquals("[::1]:80", NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("http://[::1]"))));
+ Assert.assertEquals("[::1]:443", NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("https://[::1]"))));
+ Assert.assertEquals("[::1]:1234",
+ NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("https://[::1]:1234")), 18091));
+ Assert.assertEquals("[::1]:8091",
+ NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("http://[::1]")), 8091));
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java
new file mode 100644
index 0000000..445d15f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StorageUnitTest {
+
+ @Test
+ public void test() {
+ // Valid cases
+ double result1NoUnit = StorageUtil.getSizeInBytes("1"); // Defaults to bytes
+ Assert.assertEquals(1.0, result1NoUnit, 0);
+
+ double result1B = StorageUtil.getSizeInBytes("1B");
+ Assert.assertEquals(1.0, result1B, 0);
+
+ double result1BWithSpaces = StorageUtil.getSizeInBytes("1 B ");
+ Assert.assertEquals(1.0, result1BWithSpaces, 0);
+
+ double result1Kb = StorageUtil.getSizeInBytes("1KB");
+ Assert.assertEquals(1024.0, result1Kb, 0);
+
+ double result1KbWithSpaces = StorageUtil.getSizeInBytes(" 1 K B ");
+ Assert.assertEquals(1024.0, result1KbWithSpaces, 0);
+
+ double resultPoint5KB = StorageUtil.getSizeInBytes(".5KB");
+ Assert.assertEquals(512.0, resultPoint5KB, 0);
+
+ double resultPoint5SmallKB = StorageUtil.getSizeInBytes(".5kB");
+ Assert.assertEquals(512.0, resultPoint5SmallKB, 0);
+
+ double result1Mb = StorageUtil.getSizeInBytes("1MB");
+ Assert.assertEquals(1024.0 * 1024.0, result1Mb, 0);
+
+ double result1Point0Mb = StorageUtil.getSizeInBytes("1.0MB");
+ Assert.assertEquals(1024.0 * 1024.0, result1Point0Mb, 0);
+
+ double result01Point0Mb = StorageUtil.getSizeInBytes("01.0MB");
+ Assert.assertEquals(1024.0 * 1024.0, result01Point0Mb, 0);
+
+ // Invalid cases
+ invalidCase("");
+ invalidCase("99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999");
+ invalidCase("32MB123");
+ invalidCase("1.1.1");
+ invalidCase("12KBMB");
+ invalidCase("MB");
+ invalidCase("1AB");
+ invalidCase("MB1MB");
+ invalidCase("123MBB");
+ }
+
+ private void invalidCase(String value) {
+ try {
+ StorageUtil.getSizeInBytes(value);
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.toString()
+ .contains("IllegalArgumentException: The given string: " + value + " is not a byte unit string"));
+ }
+ }
+}
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 27f9ab1..600d57c 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -71,7 +71,7 @@
<!-- Versions under dependencymanagement or used in many projects via properties -->
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.13.3</log4j.version>
+ <log4j.version>2.14.1</log4j.version>
<implementation.title>Apache Hyracks and Algebricks - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
@@ -83,7 +83,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
- <version>4.1.59.Final</version>
+ <version>4.1.63.Final</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -166,27 +166,27 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.6</version>
+ <version>2.8.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
- <version>1.8</version>
+ <version>1.9</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.12.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>2.12.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>2.12.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -218,7 +218,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.9</version>
+ <version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -228,7 +228,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.4.13</version>
+ <version>4.4.14</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>