Feed Fixes and Cleanup
1. Introduce filesystem feed data source.
2. Fix the order of closing feed stages on disconnection.
3. Added Twitter feed to the compatibility utility to allow
using it with its alias.
4. First part of the feed log space.
5. Fixed the handling of duplicate key exception.
Change-Id: I4e8db26a810efd1fbaa52ceeb3efd0c8328ab070
Reviewed-on: https://asterix-gerrit.ics.uci.edu/574
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
index 35bc3fd..fb81885 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
@@ -20,11 +20,10 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
-
import org.apache.asterix.metadata.declared.AqlDataSource;
import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
-import org.apache.asterix.metadata.declared.FeedDataSource;
import org.apache.asterix.metadata.entities.Feed;
+import org.apache.asterix.metadata.declared.FeedDataSource;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
index dab6193..1945be3 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -22,8 +22,8 @@
import java.util.List;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
import org.apache.asterix.metadata.declared.AqlDataSource;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.declared.AqlSourceId;
@@ -31,7 +31,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.base.AString;
@@ -154,7 +154,7 @@
AqlSourceId asid = new AqlSourceId(dataverse, getTargetFeed);
String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
- FeedPolicy policy = metadataProvider.findFeedPolicy(dataverse, policyName);
+ FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
if (policy == null) {
policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
if (policy == null) {
@@ -193,7 +193,7 @@
}
private AqlDataSource createFeedDataSource(AqlSourceId aqlId, String targetDataset, String sourceFeedName,
- String subscriptionLocation, AqlMetadataProvider metadataProvider, FeedPolicy feedPolicy,
+ String subscriptionLocation, AqlMetadataProvider metadataProvider, FeedPolicyEntity feedPolicy,
String outputType, String locations) throws AlgebricksException {
if (!aqlId.getDataverseName().equals(
metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse()
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index 1e24ea8..0c2463c 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -25,6 +25,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.dataflow.data.common.AqlExpressionTypeComputer;
import org.apache.asterix.metadata.api.IMetadataEntity;
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
index 87d4f1c..a161717 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
@@ -24,6 +24,7 @@
import java.util.logging.Logger;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.statement.DatasetDecl;
@@ -32,7 +33,6 @@
import org.apache.asterix.lang.common.statement.DropStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.asterix.metadata.dataset.hints.DatasetHints;
import org.apache.asterix.metadata.entities.AsterixBuiltinTypeMap;
import org.apache.asterix.metadata.entities.Dataverse;
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index d4825c4..7221972 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -22,7 +22,7 @@
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Statement.Kind;
import org.apache.asterix.lang.common.expression.VariableExpr;
diff --git a/asterix-algebra/src/main/javacc/AQLPlus.jj b/asterix-algebra/src/main/javacc/AQLPlus.jj
index 853631c..b10d837 100644
--- a/asterix-algebra/src/main/javacc/AQLPlus.jj
+++ b/asterix-algebra/src/main/javacc/AQLPlus.jj
@@ -41,6 +41,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.aql.clause.DistinctClause;
@@ -100,7 +101,6 @@
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
index 4a8a323..6d0b321 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
@@ -39,7 +39,6 @@
import org.apache.asterix.common.context.DatasetLifecycleManager;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.IFeedManager;
import org.apache.asterix.common.replication.IRemoteRecoveryManager;
import org.apache.asterix.common.replication.IReplicaResourcesManager;
import org.apache.asterix.common.replication.IReplicationChannel;
@@ -47,8 +46,9 @@
import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.management.FeedManager;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
-import org.apache.asterix.feeds.FeedManager;
import org.apache.asterix.metadata.bootstrap.MetadataIndexImmutableProperties;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.asterix.replication.management.ReplicationChannel;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
index 1e6cc66..4df461b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
@@ -29,7 +29,7 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.feeds.CentralFeedManager;
+import org.apache.asterix.feed.CentralFeedManager;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
index 35efcb1..6957926 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
@@ -32,13 +32,13 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.common.feeds.FeedActivity;
-import org.apache.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.api.IFeedLoadManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.feeds.CentralFeedManager;
+import org.apache.asterix.external.feed.api.IFeedLoadManager;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.watch.FeedActivity;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
+import org.apache.asterix.feed.CentralFeedManager;
public class FeedServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
index f1473f4..d459775 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
@@ -26,9 +26,9 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.feeds.FeedLifecycleListener;
-import org.apache.asterix.metadata.feeds.RemoteSocketMessageListener;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.RemoteSocketMessageListener;
+import org.apache.asterix.feed.FeedLifecycleListener;
public class FeedServletUtil {
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 545cb67..3b6af6d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -48,26 +48,30 @@
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedJointKey;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
-import org.apache.asterix.common.feeds.api.IFeedJoint.FeedJointType;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.api.IFeed.FeedType;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedJoint.FeedJointType;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedJointKey;
+import org.apache.asterix.external.feed.management.FeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.feeds.CentralFeedManager;
-import org.apache.asterix.feeds.FeedJoint;
-import org.apache.asterix.feeds.FeedLifecycleListener;
+import org.apache.asterix.feed.CentralFeedManager;
+import org.apache.asterix.feed.FeedJoint;
+import org.apache.asterix.feed.FeedLifecycleListener;
import org.apache.asterix.file.DatasetOperations;
import org.apache.asterix.file.DataverseOperations;
import org.apache.asterix.file.ExternalIndexingOperations;
@@ -120,7 +124,6 @@
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.api.IMetadataEntity;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.asterix.metadata.dataset.hints.DatasetHints;
import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetNodegroupCardinalityHint;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
@@ -130,16 +133,12 @@
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.Feed.FeedType;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.entities.NodeGroup;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
-import org.apache.asterix.metadata.entities.SecondaryFeed;
-import org.apache.asterix.metadata.feeds.FeedLifecycleEventSubscriber;
-import org.apache.asterix.metadata.feeds.FeedUtil;
+import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.metadata.utils.MetadataLockManager;
@@ -239,11 +238,11 @@
/**
* Compiles and submits for execution a list of AQL statements.
* @param hcc
- * A Hyracks client connection that is used to submit a jobspec to Hyracks.
+ * A Hyracks client connection that is used to submit a jobspec to Hyracks.
* @param hdc
- * A Hyracks dataset client object that is used to read the results.
+ * A Hyracks dataset client object that is used to read the results.
* @param resultDelivery
- * True if the results should be read asynchronously or false if we should wait for results to be read.
+ * True if the results should be read asynchronously or false if we should wait for results to be read.
* @return A List<QueryResult> containing a QueryResult instance corresponding to each submitted query.
* @throws Exception
*/
@@ -1933,13 +1932,13 @@
case CREATE_PRIMARY_FEED:
CreatePrimaryFeedStatement cpfs = (CreatePrimaryFeedStatement) stmt;
String adaptorName = cpfs.getAdaptorName();
- feed = new PrimaryFeed(dataverseName, feedName, adaptorName, cpfs.getAdaptorConfiguration(),
- cfs.getAppliedFunction());
+ feed = new Feed(dataverseName, feedName, cfs.getAppliedFunction(), FeedType.PRIMARY, feedName,
+ adaptorName, cpfs.getAdaptorConfiguration());
break;
case CREATE_SECONDARY_FEED:
CreateSecondaryFeedStatement csfs = (CreateSecondaryFeedStatement) stmt;
- feed = new SecondaryFeed(dataverseName, feedName, csfs.getSourceFeedName(),
- csfs.getAppliedFunction());
+ feed = new Feed(dataverseName, feedName, csfs.getAppliedFunction(), FeedType.SECONDARY,
+ csfs.getSourceFeedName(), null, null);
break;
default:
throw new IllegalStateException();
@@ -1961,14 +1960,14 @@
metadataProvider.setMetadataTxnContext(mdTxnCtx);
String dataverse;
String policy;
- FeedPolicy newPolicy = null;
+ FeedPolicyEntity newPolicy = null;
CreateFeedPolicyStatement cfps = (CreateFeedPolicyStatement) stmt;
dataverse = getActiveDataverse(null);
policy = cfps.getPolicyName();
MetadataLockManager.INSTANCE.createFeedPolicyBegin(dataverse, dataverse + "." + policy);
try {
- FeedPolicy feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(metadataProvider.getMetadataTxnContext(),
- dataverse, policy);
+ FeedPolicyEntity feedPolicy = MetadataManager.INSTANCE
+ .getFeedPolicy(metadataProvider.getMetadataTxnContext(), dataverse, policy);
if (feedPolicy != null) {
if (cfps.getIfNotExists()) {
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1980,7 +1979,7 @@
boolean extendingExisting = cfps.getSourcePolicyName() != null;
String description = cfps.getDescription() == null ? "" : cfps.getDescription();
if (extendingExisting) {
- FeedPolicy sourceFeedPolicy = MetadataManager.INSTANCE
+ FeedPolicyEntity sourceFeedPolicy = MetadataManager.INSTANCE
.getFeedPolicy(metadataProvider.getMetadataTxnContext(), dataverse, cfps.getSourcePolicyName());
if (sourceFeedPolicy == null) {
sourceFeedPolicy = MetadataManager.INSTANCE.getFeedPolicy(metadataProvider.getMetadataTxnContext(),
@@ -1991,7 +1990,7 @@
}
Map<String, String> policyProperties = sourceFeedPolicy.getProperties();
policyProperties.putAll(cfps.getProperties());
- newPolicy = new FeedPolicy(dataverse, policy, description, policyProperties);
+ newPolicy = new FeedPolicyEntity(dataverse, policy, description, policyProperties);
} else {
Properties prop = new Properties();
try {
@@ -2004,7 +2003,7 @@
for (Entry<Object, Object> entry : prop.entrySet()) {
policyProperties.put((String) entry.getKey(), (String) entry.getValue());
}
- newPolicy = new FeedPolicy(dataverse, policy, description, policyProperties);
+ newPolicy = new FeedPolicyEntity(dataverse, policy, description, policyProperties);
}
MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, newPolicy);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -2071,7 +2070,7 @@
MetadataLockManager.INSTANCE.dropFeedPolicyBegin(dataverseName, dataverseName + "." + policyName);
try {
- FeedPolicy feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(mdTxnCtx, dataverseName, policyName);
+ FeedPolicyEntity feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(mdTxnCtx, dataverseName, policyName);
if (feedPolicy == null) {
if (!stmtFeedPolicyDrop.getIfExists()) {
throw new AlgebricksException("Unknown policy " + policyName + " in dataverse " + dataverseName);
@@ -2112,10 +2111,10 @@
CompiledConnectFeedStatement cbfs = new CompiledConnectFeedStatement(dataverseName, cfs.getFeedName(),
cfs.getDatasetName().getValue(), cfs.getPolicy(), cfs.getQuery(), cfs.getVarCounter());
- FeedUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(),
+ FeedMetadataUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(),
metadataProvider.getMetadataTxnContext());
- Feed feed = FeedUtil.validateIfFeedExists(dataverseName, cfs.getFeedName(),
+ Feed feed = FeedMetadataUtil.validateIfFeedExists(dataverseName, cfs.getFeedName(),
metadataProvider.getMetadataTxnContext());
feedConnId = new FeedConnectionId(dataverseName, cfs.getFeedName(), cfs.getDatasetName().getValue());
@@ -2125,7 +2124,8 @@
+ cfs.getDatasetName().getValue());
}
- FeedPolicy feedPolicy = FeedUtil.validateIfPolicyExists(dataverseName, cbfs.getPolicyName(), mdTxnCtx);
+ FeedPolicyEntity feedPolicy = FeedMetadataUtil.validateIfPolicyExists(dataverseName, cbfs.getPolicyName(),
+ mdTxnCtx);
// All Metadata checks have passed. Feed connect request is valid. //
@@ -2139,8 +2139,8 @@
subscriberRegistered = true;
if (createFeedIntakeJob) {
FeedId feedId = connectionRequest.getFeedJointKey().getFeedId();
- PrimaryFeed primaryFeed = (PrimaryFeed) MetadataManager.INSTANCE.getFeed(mdTxnCtx,
- feedId.getDataverse(), feedId.getFeedName());
+ Feed primaryFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, feedId.getDataverse(),
+ feedId.getFeedName());
Pair<JobSpecification, IAdapterFactory> pair = FeedOperations.buildFeedIntakeJobSpec(primaryFeed,
metadataProvider, policyAccessor);
// adapter configuration are valid at this stage
@@ -2205,7 +2205,7 @@
* @throws MetadataException
*/
private Triple<FeedConnectionRequest, Boolean, List<IFeedJoint>> getFeedConnectionRequest(String dataverse,
- Feed feed, String dataset, FeedPolicy feedPolicy, MetadataTransactionContext mdTxnCtx)
+ Feed feed, String dataset, FeedPolicyEntity feedPolicy, MetadataTransactionContext mdTxnCtx)
throws MetadataException {
IFeedJoint sourceFeedJoint = null;
FeedConnectionRequest request = null;
@@ -2269,12 +2269,12 @@
private FeedJointKey getFeedJointKey(Feed feed, MetadataTransactionContext ctx) throws MetadataException {
Feed sourceFeed = feed;
List<String> appliedFunctions = new ArrayList<String>();
- while (sourceFeed.getFeedType().equals(FeedType.SECONDARY)) {
+ while (sourceFeed.getFeedType().equals(IFeed.FeedType.SECONDARY)) {
if (sourceFeed.getAppliedFunction() != null) {
appliedFunctions.add(0, sourceFeed.getAppliedFunction().getName());
}
Feed parentFeed = MetadataManager.INSTANCE.getFeed(ctx, feed.getDataverseName(),
- ((SecondaryFeed) sourceFeed).getSourceFeedName());
+ sourceFeed.getSourceFeedName());
sourceFeed = parentFeed;
}
@@ -2295,8 +2295,8 @@
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
- FeedUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(), mdTxnCtx);
- Feed feed = FeedUtil.validateIfFeedExists(dataverseName, cfs.getFeedName().getValue(), mdTxnCtx);
+ FeedMetadataUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(), mdTxnCtx);
+ Feed feed = FeedMetadataUtil.validateIfFeedExists(dataverseName, cfs.getFeedName().getValue(), mdTxnCtx);
FeedConnectionId connectionId = new FeedConnectionId(feed.getFeedId(), cfs.getDatasetName().getValue());
boolean isFeedConnectionActive = FeedLifecycleListener.INSTANCE.isFeedConnectionActive(connectionId);
@@ -2369,7 +2369,7 @@
try {
- JobSpecification alteredJobSpec = FeedUtil.alterJobSpecificationForFeed(compiled, feedConnectionId,
+ JobSpecification alteredJobSpec = FeedMetadataUtil.alterJobSpecificationForFeed(compiled, feedConnectionId,
bfs.getSubscriptionRequest().getPolicyParameters());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/CentralFeedManager.java b/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
similarity index 93%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/CentralFeedManager.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
index 7326d08..4020bde 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/CentralFeedManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.io.IOException;
import java.io.PrintWriter;
@@ -27,16 +27,16 @@
import org.apache.asterix.api.common.SessionConfig.OutputFormat;
import org.apache.asterix.aql.translator.QueryTranslator;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.ICentralFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedLoadManager;
-import org.apache.asterix.common.feeds.api.IFeedTrackingManager;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.external.feed.api.ICentralFeedManager;
+import org.apache.asterix.external.feed.api.IFeedLoadManager;
+import org.apache.asterix.external.feed.api.IFeedTrackingManager;
+import org.apache.asterix.external.feed.message.SocketMessageListener;
import org.apache.asterix.lang.aql.parser.AQLParserFactory;
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.metadata.feeds.SocketMessageListener;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.job.JobId;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedJobNotificationHandler.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java
similarity index 94%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedJobNotificationHandler.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java
index aa7388e..b42ef1e 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedJobNotificationHandler.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.rmi.RemoteException;
import java.util.ArrayList;
@@ -33,32 +33,31 @@
import java.util.logging.Logger;
import org.apache.commons.lang3.StringUtils;
-
-import org.apache.asterix.api.common.FeedWorkCollection.SubscribeFeedWork;
import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.feeds.FeedActivity;
-import org.apache.asterix.common.feeds.FeedConnectJobInfo;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedIntakeInfo;
-import org.apache.asterix.common.feeds.FeedJobInfo;
-import org.apache.asterix.common.feeds.FeedJobInfo.FeedJobState;
-import org.apache.asterix.common.feeds.FeedJobInfo.JobType;
-import org.apache.asterix.common.feeds.FeedJointKey;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
-import org.apache.asterix.common.feeds.api.IFeedJoint.State;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.common.feeds.message.StorageReportFeedMessage;
-import org.apache.asterix.feeds.FeedLifecycleListener.Message;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.api.IIntakeProgressTracker;
+import org.apache.asterix.external.feed.api.IFeedJoint.State;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedJointKey;
+import org.apache.asterix.external.feed.management.FeedWorkManager;
+import org.apache.asterix.external.feed.message.StorageReportFeedMessage;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.watch.FeedActivity;
+import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
+import org.apache.asterix.external.feed.watch.FeedIntakeInfo;
+import org.apache.asterix.external.feed.watch.FeedJobInfo;
+import org.apache.asterix.external.feed.watch.FeedJobInfo.FeedJobState;
+import org.apache.asterix.external.feed.watch.FeedJobInfo.JobType;
+import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
+import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
+import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor;
+import org.apache.asterix.feed.FeedLifecycleListener.Message;
+import org.apache.asterix.feed.FeedWorkCollection.SubscribeFeedWork;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
-import org.apache.asterix.metadata.feeds.FeedCollectOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.FeedIntakeOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.FeedMetaOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.FeedWorkManager;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedJoint.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java
similarity index 92%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedJoint.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java
index a76a1e9..43f227d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedJoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java
@@ -16,19 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedJointKey;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedJointKey;
public class FeedJoint implements IFeedJoint {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedLifecycleListener.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java
similarity index 93%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedLifecycleListener.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java
index d53428d..aac3675 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedLifecycleListener.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.io.PrintWriter;
import java.util.ArrayList;
@@ -39,21 +39,24 @@
import org.apache.asterix.common.api.IClusterManagementWork;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.api.IClusterManagementWorkResponse;
-import org.apache.asterix.common.feeds.FeedConnectJobInfo;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedIntakeInfo;
-import org.apache.asterix.common.feeds.FeedJobInfo;
-import org.apache.asterix.common.feeds.FeedJobInfo.FeedJobState;
-import org.apache.asterix.common.feeds.FeedJointKey;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.common.feeds.message.StorageReportFeedMessage;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener;
+import org.apache.asterix.external.feed.api.IIntakeProgressTracker;
+import org.apache.asterix.external.feed.management.FeedCollectInfo;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedJointKey;
+import org.apache.asterix.external.feed.message.StorageReportFeedMessage;
+import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
+import org.apache.asterix.external.feed.watch.FeedIntakeInfo;
+import org.apache.asterix.external.feed.watch.FeedJobInfo;
+import org.apache.asterix.external.feed.watch.FeedJobInfo.FeedJobState;
+import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
+import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
@@ -62,8 +65,6 @@
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.cluster.AddNodeWork;
import org.apache.asterix.metadata.cluster.ClusterManager;
-import org.apache.asterix.metadata.feeds.FeedCollectOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.FeedIntakeOperatorDescriptor;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -324,7 +325,8 @@
case ACTIVE:
if (previousState.equals(ClusterState.UNUSABLE)) {
try {
- FeedsActivator activator = new FeedsActivator();
+ // TODO: Figure out why code was commented
+ // FeedsActivator activator = new FeedsActivator();
// (new Thread(activator)).start();
} catch (Exception e) {
if (LOGGER.isLoggable(Level.INFO)) {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedLoadManager.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java
similarity index 90%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedLoadManager.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java
index cb3133e..ee4da11 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedLoadManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.util.ArrayList;
import java.util.Collection;
@@ -31,22 +31,22 @@
import java.util.logging.Logger;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedActivity;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedJobInfo.FeedJobState;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.NodeLoadReport;
-import org.apache.asterix.common.feeds.api.IFeedLoadManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedTrackingManager;
-import org.apache.asterix.common.feeds.message.FeedCongestionMessage;
-import org.apache.asterix.common.feeds.message.FeedReportMessage;
-import org.apache.asterix.common.feeds.message.ScaleInReportMessage;
-import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedLoadManager;
+import org.apache.asterix.external.feed.api.IFeedTrackingManager;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.FeedCongestionMessage;
+import org.apache.asterix.external.feed.message.FeedReportMessage;
+import org.apache.asterix.external.feed.message.PrepareStallMessage;
+import org.apache.asterix.external.feed.message.ScaleInReportMessage;
+import org.apache.asterix.external.feed.message.TerminateDataFlowMessage;
+import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.feed.watch.FeedActivity;
+import org.apache.asterix.external.feed.watch.NodeLoadReport;
+import org.apache.asterix.external.feed.watch.FeedJobInfo.FeedJobState;
import org.apache.asterix.file.FeedOperations;
-import org.apache.asterix.metadata.feeds.FeedUtil;
-import org.apache.asterix.metadata.feeds.PrepareStallMessage;
-import org.apache.asterix.metadata.feeds.TerminateDataFlowMessage;
+import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.client.IHyracksClientConnection;
@@ -117,7 +117,7 @@
List<String> newLocations = new ArrayList<String>();
newLocations.addAll(currentComputeLocations);
newLocations.addAll(helperComputeNodes);
- FeedUtil.increaseCardinality(jobSpec, FeedRuntimeType.COMPUTE, requiredCardinality, newLocations);
+ FeedMetadataUtil.increaseCardinality(jobSpec, FeedRuntimeType.COMPUTE, requiredCardinality, newLocations);
// Step 2) send prepare to stall message
gracefullyTerminateDataFlow(message.getConnectionId(), Integer.MAX_VALUE);
@@ -161,7 +161,7 @@
List<String> currentComputeLocations = new ArrayList<String>();
currentComputeLocations.addAll(FeedLifecycleListener.INSTANCE.getComputeLocations(message.getConnectionId()
.getFeedId()));
- FeedUtil.decreaseComputeCardinality(jobSpec, FeedRuntimeType.COMPUTE, reducedCardinality,
+ FeedMetadataUtil.decreaseComputeCardinality(jobSpec, FeedRuntimeType.COMPUTE, reducedCardinality,
currentComputeLocations);
gracefullyTerminateDataFlow(message.getConnectionId(), reducedCardinality - 1);
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedMessageReceiver.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java
similarity index 77%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedMessageReceiver.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java
index a3cd217..66eca0c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedMessageReceiver.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java
@@ -16,25 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.util.logging.Level;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedTupleCommitAckMessage;
-import org.apache.asterix.common.feeds.MessageReceiver;
-import org.apache.asterix.common.feeds.NodeLoadReport;
-import org.apache.asterix.common.feeds.api.IFeedLoadManager;
-import org.apache.asterix.common.feeds.api.IFeedMessage.MessageType;
-import org.apache.asterix.common.feeds.api.IFeedTrackingManager;
-import org.apache.asterix.common.feeds.message.FeedCongestionMessage;
-import org.apache.asterix.common.feeds.message.FeedReportMessage;
-import org.apache.asterix.common.feeds.message.ScaleInReportMessage;
-import org.apache.asterix.common.feeds.message.StorageReportFeedMessage;
-import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
-import org.apache.asterix.feeds.CentralFeedManager.AQLExecutor;
+import org.apache.asterix.external.feed.api.IFeedLoadManager;
+import org.apache.asterix.external.feed.api.IFeedTrackingManager;
+import org.apache.asterix.external.feed.api.IFeedMessage.MessageType;
+import org.apache.asterix.external.feed.message.FeedCongestionMessage;
+import org.apache.asterix.external.feed.message.FeedReportMessage;
+import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
+import org.apache.asterix.external.feed.message.MessageReceiver;
+import org.apache.asterix.external.feed.message.ScaleInReportMessage;
+import org.apache.asterix.external.feed.message.StorageReportFeedMessage;
+import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.watch.NodeLoadReport;
+import org.apache.asterix.external.util.FeedConstants;
+import org.apache.asterix.feed.CentralFeedManager.AQLExecutor;
import org.apache.asterix.hyracks.bootstrap.FeedBootstrap;
public class FeedMessageReceiver extends MessageReceiver<String> {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedTrackingManager.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java
similarity index 95%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedTrackingManager.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java
index d57a971..b65d8b1 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedTrackingManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.util.Arrays;
import java.util.BitSet;
@@ -28,10 +28,10 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedTupleCommitAckMessage;
-import org.apache.asterix.common.feeds.FeedTupleCommitResponseMessage;
-import org.apache.asterix.common.feeds.api.IFeedTrackingManager;
+import org.apache.asterix.external.feed.api.IFeedTrackingManager;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
+import org.apache.asterix.external.feed.message.FeedTupleCommitResponseMessage;
import org.apache.asterix.file.FeedOperations;
import org.apache.hyracks.api.job.JobSpecification;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java
similarity index 94%
rename from asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java
index 6aad64b..9d746c8 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.api.common;
+package org.apache.asterix.feed;
import java.io.PrintWriter;
import java.util.ArrayList;
@@ -24,15 +24,16 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.api.common.SessionConfig;
import org.apache.asterix.api.common.SessionConfig.OutputFormat;
import org.apache.asterix.aql.translator.QueryTranslator;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedConnectionRequest.ConnectionStatus;
-import org.apache.asterix.common.feeds.api.IFeedWork;
-import org.apache.asterix.common.feeds.api.IFeedWorkEventListener;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.feeds.FeedCollectInfo;
+import org.apache.asterix.external.feed.api.IFeedWork;
+import org.apache.asterix.external.feed.api.IFeedWorkEventListener;
+import org.apache.asterix.external.feed.management.FeedCollectInfo;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest.ConnectionStatus;
import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.statement.DataverseDecl;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedWorkRequestResponseHandler.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java
similarity index 98%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedWorkRequestResponseHandler.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java
index 3686a03..b30d8a7 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedWorkRequestResponseHandler.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.util.ArrayList;
import java.util.HashMap;
@@ -31,9 +31,9 @@
import org.apache.asterix.common.api.IClusterManagementWork;
import org.apache.asterix.common.api.IClusterManagementWorkResponse;
-import org.apache.asterix.common.feeds.FeedConnectJobInfo;
-import org.apache.asterix.common.feeds.FeedIntakeInfo;
-import org.apache.asterix.common.feeds.FeedJobInfo;
+import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
+import org.apache.asterix.external.feed.watch.FeedIntakeInfo;
+import org.apache.asterix.external.feed.watch.FeedJobInfo;
import org.apache.asterix.metadata.cluster.AddNodeWork;
import org.apache.asterix.metadata.cluster.AddNodeWorkResponse;
import org.apache.asterix.om.util.AsterixAppContextInfo;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedsActivator.java b/asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java
similarity index 97%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedsActivator.java
rename to asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java
index 7660007..1a8a119 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedsActivator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.feed;
import java.io.PrintWriter;
import java.util.ArrayList;
@@ -29,6 +29,7 @@
import org.apache.asterix.aql.translator.QueryTranslator;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.external.feed.management.FeedCollectInfo;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.DataverseDecl;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
index 2ee9dd4..77c6a54 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
@@ -44,6 +44,7 @@
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.FilesIndexDescription;
import org.apache.asterix.external.indexing.IndexingConstants;
+import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalDatasetIndexesAbortOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalDatasetIndexesCommitOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalDatasetIndexesRecoverOperatorDescriptor;
@@ -60,7 +61,6 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.om.types.ARecordType;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
index cb55c5f..6a036c0 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
@@ -22,24 +22,24 @@
import java.util.List;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedConnectJobInfo;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.FeedTupleCommitResponseMessage;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
-import org.apache.asterix.common.feeds.api.IFeedMessage;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.message.EndFeedMessage;
-import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
import org.apache.asterix.external.api.IAdapterFactory;
-import org.apache.asterix.feeds.FeedLifecycleListener;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.message.EndFeedMessage;
+import org.apache.asterix.external.feed.message.FeedTupleCommitResponseMessage;
+import org.apache.asterix.external.feed.message.PrepareStallMessage;
+import org.apache.asterix.external.feed.message.TerminateDataFlowMessage;
+import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
+import org.apache.asterix.external.operators.FeedMessageOperatorDescriptor;
+import org.apache.asterix.external.util.FeedConstants;
+import org.apache.asterix.feed.FeedLifecycleListener;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
-import org.apache.asterix.metadata.feeds.FeedMessageOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.PrepareStallMessage;
-import org.apache.asterix.metadata.feeds.TerminateDataFlowMessage;
+import org.apache.asterix.metadata.entities.Feed;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
@@ -63,7 +63,7 @@
* @return JobSpecification the Hyracks job specification for receiving data from external source
* @throws Exception
*/
- public static Pair<JobSpecification, IAdapterFactory> buildFeedIntakeJobSpec(PrimaryFeed primaryFeed,
+ public static Pair<JobSpecification, IAdapterFactory> buildFeedIntakeJobSpec(Feed primaryFeed,
AqlMetadataProvider metadataProvider, FeedPolicyAccessor policyAccessor) throws Exception {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
index a579d2c..6d23f3c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -31,9 +31,9 @@
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
import org.apache.asterix.external.indexing.IndexingConstants;
+import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index 44af0ff..d67ca0d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -38,6 +38,7 @@
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingConstants;
+import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalIndexBulkModifyOperatorDescriptor;
import org.apache.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
@@ -46,7 +47,6 @@
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
index 4d887dc..418d143 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -31,12 +31,12 @@
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
import org.apache.asterix.external.indexing.IndexingConstants;
+import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
index c6e04df..4fae7e9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
@@ -26,15 +26,15 @@
import org.apache.asterix.common.api.IClusterEventsSubscriber;
import org.apache.asterix.common.api.IClusterManagementWork;
import org.apache.asterix.common.api.IClusterManagementWorkResponse;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.feeds.CentralFeedManager;
+import org.apache.asterix.feed.CentralFeedManager;
import org.apache.asterix.file.ExternalIndexingOperations;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Dataverse;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index d2164f4..2a7b3e4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -37,12 +37,12 @@
import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.common.config.AsterixMetadataProperties;
import org.apache.asterix.common.config.AsterixReplicationProperties;
-import org.apache.asterix.common.feeds.api.ICentralFeedManager;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.event.service.ILookupService;
-import org.apache.asterix.feeds.CentralFeedManager;
-import org.apache.asterix.feeds.FeedLifecycleListener;
+import org.apache.asterix.external.feed.api.ICentralFeedManager;
+import org.apache.asterix.feed.CentralFeedManager;
+import org.apache.asterix.feed.FeedLifecycleListener;
import org.apache.asterix.messaging.CCMessageBroker;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.api.IAsterixStateProxy;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
index 01775ab..b0dfd58 100755
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
@@ -36,6 +36,8 @@
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.external.api.IDataSourceAdapter;
+import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
import org.apache.asterix.external.library.ExternalLibrary;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.external.library.LibraryAdapter;
@@ -44,10 +46,8 @@
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.api.IMetadataEntity;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
-import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Library;
-import org.apache.asterix.metadata.feeds.AdapterIdentifier;
import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
public class ExternalLibraryBootstrap {
@@ -210,7 +210,8 @@
String adapterFactoryClass = adapter.getFactoryClass().trim();
String adapterName = libraryName + "#" + adapter.getName().trim();
AdapterIdentifier aid = new AdapterIdentifier(dataverse, adapterName);
- DatasourceAdapter dsa = new DatasourceAdapter(aid, adapterFactoryClass, AdapterType.EXTERNAL);
+ DatasourceAdapter dsa = new DatasourceAdapter(aid, adapterFactoryClass,
+ IDataSourceAdapter.AdapterType.EXTERNAL);
MetadataManager.INSTANCE.addAdapter(mdTxnCtx, dsa);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Installed adapter: " + adapterName);
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
index 2d443c7..d5f1a51 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
@@ -18,29 +18,23 @@
*/
package org.apache.asterix.hyracks.bootstrap;
-import org.apache.asterix.feeds.CentralFeedManager;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
+import org.apache.asterix.common.config.MetadataConstants;
+import org.apache.asterix.external.util.FeedConstants;
+import org.apache.asterix.feed.CentralFeedManager;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
public class FeedBootstrap {
- public final static String FEEDS_METADATA_DV = "feeds_metadata";
- public final static String FAILED_TUPLE_DATASET = "failed_tuple";
- public final static String FAILED_TUPLE_DATASET_TYPE = "FailedTupleType";
- public final static String FAILED_TUPLE_DATASET_KEY = "id";
-
public static void setUpInitialArtifacts() throws Exception {
StringBuilder builder = new StringBuilder();
try {
- builder.append("create dataverse " + FEEDS_METADATA_DV + ";" + "\n");
- builder.append("use dataverse " + FEEDS_METADATA_DV + ";" + "\n");
-
- builder.append("create type " + FAILED_TUPLE_DATASET_TYPE + " as open { ");
-
- String[] fieldNames = new String[] { "id", "dataverseName", "feedName", "targetDataset", "tuple",
- "message", "timestamp" };
+ builder.append("create dataverse " + FeedConstants.FEEDS_METADATA_DV + ";" + "\n");
+ builder.append("use dataverse " + FeedConstants.FEEDS_METADATA_DV + ";" + "\n");
+ builder.append("create type " + FeedConstants.FAILED_TUPLE_DATASET_TYPE + " as open { ");
+ String[] fieldNames = new String[] { "id", "dataverseName", "feedName", "targetDataset", "tuple", "message",
+ "timestamp" };
IAType[] fieldTypes = new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING,
BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING };
@@ -52,9 +46,9 @@
builder.append(fieldTypes[i].getTypeName());
}
builder.append("}" + ";" + "\n");
-
- builder.append("create dataset " + FAILED_TUPLE_DATASET + " " + "(" + FAILED_TUPLE_DATASET_TYPE + ")" + " "
- + "primary key " + FAILED_TUPLE_DATASET_KEY + " on " + MetadataConstants.METADATA_NODEGROUP_NAME
+ builder.append("create dataset " + FeedConstants.FAILED_TUPLE_DATASET + " " + "("
+ + FeedConstants.FAILED_TUPLE_DATASET_TYPE + ")" + " " + "primary key "
+ + FeedConstants.FAILED_TUPLE_DATASET_KEY + " on " + MetadataConstants.METADATA_NODEGROUP_NAME
+ ";");
CentralFeedManager.AQLExecutor.executeAQL(builder.toString());
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 76e7856..3341387 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -38,6 +38,7 @@
import org.apache.asterix.common.replication.IRemoteRecoveryManager;
import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.event.schema.cluster.Cluster;
import org.apache.asterix.event.schema.cluster.Node;
import org.apache.asterix.messaging.NCMessageBroker;
@@ -46,7 +47,6 @@
import org.apache.asterix.metadata.api.IAsterixStateProxy;
import org.apache.asterix.metadata.api.IMetadataNode;
import org.apache.asterix.metadata.bootstrap.MetadataBootstrap;
-import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.asterix.transaction.management.service.recovery.RecoveryManager;
@@ -114,7 +114,7 @@
runtimeContext.initialize(initialRun);
ncApplicationContext.setApplicationObject(runtimeContext);
- //if replication is enabled, check if there is a replica for this node
+ //If replication is enabled, check if there is a replica for this node
AsterixReplicationProperties asterixReplicationProperties = ((IAsterixPropertiesProvider) runtimeContext)
.getReplicationProperties();
@@ -123,7 +123,7 @@
if (initialRun) {
LOGGER.info("System is being initialized. (first run)");
} else {
- // #. recover if the system is corrupted by checking system state.
+ //#. recover if the system is corrupted by checking system state.
IRecoveryManager recoveryMgr = runtimeContext.getTransactionSubsystem().getRecoveryManager();
systemState = recoveryMgr.getSystemState();
@@ -133,7 +133,7 @@
if (replicationEnabled) {
if (systemState == SystemState.NEW_UNIVERSE || systemState == SystemState.CORRUPTED) {
- //try to perform remote recovery
+ //Try to perform remote recovery
IRemoteRecoveryManager remoteRecoveryMgr = runtimeContext.getRemoteRecoveryManager();
remoteRecoveryMgr.performRemoteRecovery();
performedRemoteRecovery = true;
@@ -152,20 +152,20 @@
}
private void startReplicationService() throws IOException {
- //open replication channel
+ //Open replication channel
runtimeContext.getReplicationChannel().start();
- //check the state of remote replicas
+ //Check the state of remote replicas
runtimeContext.getReplicationManager().initializeReplicasState();
if (performedRemoteRecovery) {
- //notify remote replicas about the new IP Address if changed
+ //Notify remote replicas about the new IP Address if changed
//Note: this is a hack since each node right now maintains its own copy of the cluster configuration.
//Once the configuration is centralized on the CC, this step wont be needed.
runtimeContext.getReplicationManager().broadcastNewIPAddress();
}
- //start replication after the state of remote replicas has been initialized.
+ //Start replication after the state of remote replicas has been initialized.
runtimeContext.getReplicationManager().startReplicationThreads();
}
@@ -182,7 +182,7 @@
MetadataBootstrap.stopUniverse();
}
- //clean any temporary files
+ //Clean any temporary files
performLocalCleanUp();
//Note: stopping recovery manager will make a sharp checkpoint
@@ -197,7 +197,7 @@
@Override
public void notifyStartupComplete() throws Exception {
- //send max resource id on this NC to the CC
+ //Send max resource id on this NC to the CC
((INCMessageBroker) ncApplicationContext.getMessageBroker()).reportMaxResourceId();
AsterixMetadataProperties metadataProperties = ((IAsterixPropertiesProvider) runtimeContext)
@@ -228,9 +228,9 @@
throw new IllegalStateException("Metadata node cannot access distributed state");
}
- // This is a special case, we just give the metadataNode directly.
- // This way we can delay the registration of the metadataNode until
- // it is completely initialized.
+ //This is a special case, we just give the metadataNode directly.
+ //This way we can delay the registration of the metadataNode until
+ //it is completely initialized.
MetadataManager.INSTANCE = new MetadataManager(proxy, MetadataNode.INSTANCE);
MetadataBootstrap.startUniverse(((IAsterixPropertiesProvider) runtimeContext), ncApplicationContext,
systemState == SystemState.NEW_UNIVERSE);
@@ -272,26 +272,26 @@
proxy.setMetadataNode(stub);
}
- //clean any temporary files
+ //Clean any temporary files
performLocalCleanUp();
}
private void performLocalCleanUp() {
- //delete working area files from failed jobs
+ //Delete working area files from failed jobs
runtimeContext.getIOManager().deleteWorkspaceFiles();
- //reclaim storage for temporary datasets.
+ //Reclaim storage for temporary datasets.
String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
String[] ioDevices = ((PersistentLocalResourceRepository) runtimeContext.getLocalResourceRepository())
.getStorageMountingPoints();
for (String ioDevice : ioDevices) {
String tempDatasetsDir = ioDevice + storageDirName + File.separator
- + SplitsAndConstraintsUtil.TEMP_DATASETS_STORAGE_FOLDER;
+ + StoragePathUtil.TEMP_DATASETS_STORAGE_FOLDER;
FileUtils.deleteQuietly(new File(tempDatasetsDir));
}
- // TODO
- //reclaim storage for orphaned index artifacts in NCs.
+ //TODO
+ //Reclaim storage for orphaned index artifacts in NCs.
//Note: currently LSM indexes invalid components are deleted when an index is activated.
}
@@ -321,7 +321,7 @@
String nodeIoDevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
String[] ioDevicePaths = nodeIoDevices.trim().split(",");
for (int i = 0; i < ioDevicePaths.length; i++) {
- //construct full store path
+ // construct full store path
ioDevicePaths[i] += File.separator + storeDir;
}
metadataProperties.getStores().put(nodeId, ioDevicePaths);
diff --git a/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java b/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
index 8ff6d9b..83beb7c 100644
--- a/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
+++ b/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
@@ -36,7 +36,7 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.feeds.CentralFeedManager;
+import org.apache.asterix.feed.CentralFeedManager;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
index c4dde05..aabf05d 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
@@ -1,3 +1 @@
-{ "DataverseName": "Metadata", "Name": "adapter", "Classname": "org.apache.asterix.external.adapter.factory.GenericAdapterFactory", "Type": "INTERNAL", "Timestamp": "Sun Jan 03 15:39:35 AST 2016" }
-{ "DataverseName": "Metadata", "Name": "socket_adapter", "Classname": "org.apache.asterix.external.runtime.GenericSocketFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Sun Jan 03 15:39:35 AST 2016" }
-{ "DataverseName": "Metadata", "Name": "socket_client", "Classname": "org.apache.asterix.external.runtime.SocketClientAdapterFactory", "Type": "INTERNAL", "Timestamp": "Sun Jan 03 15:39:35 AST 2016" }
\ No newline at end of file
+{ "DataverseName": "Metadata", "Name": "adapter", "Classname": "org.apache.asterix.external.adapter.factory.GenericAdapterFactory", "Type": "INTERNAL", "Timestamp": "Sun Jan 10 16:13:18 AST 2016" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm b/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
index f981aca..f3e4605 100644
--- a/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
@@ -1 +1 @@
-{ "DataverseName": "feeds", "FeedName": "TweetFeed", "Function": null, "FeedType": "PRIMARY", "PrimaryTypeDetails": { "AdapterName": "file_feed", "AdapterConfiguration": {{ { "Name": "output-type-name", "Value": "TweetType" }, { "Name": "fs", "Value": "localfs" }, { "Name": "path", "Value": "nc1://data/twitter/obamatweets.adm" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" } }} }, "SecondaryTypeDetails": null, "Timestamp": "Sat Jun 20 13:55:58 PDT 2015" }
+{ "DataverseName": "feeds", "FeedName": "TweetFeed", "Function": null, "FeedType": "PRIMARY", "PrimaryTypeDetails": { "AdapterName": "file_feed", "AdapterConfiguration": {{ { "Name": "output-type-name", "Value": "TweetType" }, { "Name": "fs", "Value": "localfs" }, { "Name": "path", "Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" } }} }, "SecondaryTypeDetails": null, "Timestamp": "Sat Jun 20 13:55:58 PDT 2015" }
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index e14b558..375d05a 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -27,6 +27,82 @@
ResultOffsetPath="results"
QueryOffsetPath="queries"
QueryFileExtension=".aql">
+ <test-group name="feeds">
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_01">
+ <output-dir compare="Text">feeds_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!--Disable it because of sporadic failures. Abdullah will re-enable it.
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_02">
+ <output-dir compare="Text">feeds_02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_03">
+ <output-dir compare="Text">feeds_03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_04">
+ <output-dir compare="Text">feeds_04</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_06">
+ <output-dir compare="Text">feeds_06</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_07">
+ <output-dir compare="Text">feeds_07</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_08">
+ <output-dir compare="Text">feeds_08</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_09">
+ <output-dir compare="Text">feeds_09</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_10">
+ <output-dir compare="Text">feeds_10</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_11">
+ <output-dir compare="Text">feeds_11</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="feeds_12">
+ <output-dir compare="Text">feeds_12</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="feeds">
+ <compilation-unit name="issue_230_feeds">
+ <output-dir compare="Text">issue_230_feeds</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="feeds">
+ <compilation-unit name="issue_711_feeds">
+ <output-dir compare="Text">issue_711_feeds</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+
+ </test-group>
<test-group name="flwor">
<test-case FilePath="flwor">
<compilation-unit name="at00">
@@ -6137,83 +6213,6 @@
</compilation-unit>
</test-case>
</test-group>
- <test-group name="feeds">
-
- <!--Disable it because of sporadic failures. Raman will re-enable it.
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_01">
- <output-dir compare="Text">feeds_01</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_02">
- <output-dir compare="Text">feeds_02</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_03">
- <output-dir compare="Text">feeds_03</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_04">
- <output-dir compare="Text">feeds_04</output-dir>
- </compilation-unit>
- </test-case>
-
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_06">
- <output-dir compare="Text">feeds_06</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_07">
- <output-dir compare="Text">feeds_07</output-dir>
- </compilation-unit>
- </test-case>
-
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_08">
- <output-dir compare="Text">feeds_08</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_09">
- <output-dir compare="Text">feeds_09</output-dir>
- </compilation-unit>
- </test-case>
-
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_10">
- <output-dir compare="Text">feeds_10</output-dir>
- </compilation-unit>
- </test-case>
-
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_11">
- <output-dir compare="Text">feeds_11</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="feeds">
- <compilation-unit name="feeds_12">
- <output-dir compare="Text">feeds_12</output-dir>
- </compilation-unit>
- </test-case>
-
- <test-case FilePath="feeds">
- <compilation-unit name="issue_230_feeds">
- <output-dir compare="Text">issue_230_feeds</output-dir>
- </compilation-unit>
- </test-case>
-
- <test-case FilePath="feeds">
- <compilation-unit name="issue_711_feeds">
- <output-dir compare="Text">issue_711_feeds</output-dir>
- </compilation-unit>
- </test-case>
- -->
-
- </test-group>
<test-group name="hdfs">
<test-case FilePath="hdfs">
<compilation-unit name="large-record">
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
index b8c3f2f..3386252 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -24,7 +24,6 @@
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.IFeedManager;
import org.apache.asterix.common.replication.IRemoteRecoveryManager;
import org.apache.asterix.common.replication.IReplicaResourcesManager;
import org.apache.asterix.common.replication.IReplicationChannel;
@@ -79,7 +78,7 @@
public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
- public IFeedManager getFeedManager();
+ public Object getFeedManager();
public IRemoteRecoveryManager getRemoteRecoveryManager();
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java b/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java
index cc7ec84..13ce403 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java
@@ -197,7 +197,7 @@
}
public ClusterPartition getMetadataPartiton() {
- //metadata partition is always the first partition on the metadata node
+ // metadata partition is always the first partition on the metadata node
return nodePartitionsMap.get(metadataNodeName)[0];
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataConstants.java b/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataConstants.java
similarity index 96%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataConstants.java
rename to asterix-common/src/main/java/org/apache/asterix/common/config/MetadataConstants.java
index a68ea81..943e385 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataConstants.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataConstants.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.asterix.metadata.bootstrap;
+package org.apache.asterix.common.config;
public class MetadataConstants {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java b/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
index fd1ebb8..d25e51f 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
@@ -21,6 +21,7 @@
import java.nio.ByteBuffer;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
+import org.apache.asterix.common.exceptions.FrameDataException;
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
@@ -41,6 +42,7 @@
private final boolean isPrimary;
private AbstractLSMIndex lsmIndex;
+ private int i = 0;
public boolean isPrimary() {
return isPrimary;
@@ -85,7 +87,7 @@
ILSMIndexAccessor lsmAccessor = (ILSMIndexAccessor) indexAccessor;
int tupleCount = accessor.getTupleCount();
try {
- for (int i = 0; i < tupleCount; i++) {
+ for (; i < tupleCount; i++) {
if (tupleFilter != null) {
frameTuple.reset(accessor, i);
if (!tupleFilter.accept(frameTuple)) {
@@ -117,11 +119,13 @@
}
}
} catch (Throwable th) {
- throw new HyracksDataException(th);
+ FrameDataException fde = new FrameDataException(i, th);
+ throw fde;
}
writeBuffer.ensureFrameSize(buffer.capacity());
FrameUtils.copyAndFlip(buffer, writeBuffer.getBuffer());
FrameUtils.flushFrame(writeBuffer.getBuffer(), writer);
+ i = 0;
}
@Override
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/exceptions/FrameDataException.java b/asterix-common/src/main/java/org/apache/asterix/common/exceptions/FrameDataException.java
index 136a196..18b5264 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/exceptions/FrameDataException.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/exceptions/FrameDataException.java
@@ -26,7 +26,7 @@
private final int tupleIndex;
- public FrameDataException(int tupleIndex, Exception cause) {
+ public FrameDataException(int tupleIndex, Throwable cause) {
super(cause);
this.tupleIndex = tupleIndex;
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeReport.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeReport.java
deleted file mode 100644
index d7717ac..0000000
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeReport.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.feeds;
-
-public class FeedRuntimeReport {
-
-}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ITupleTrackingFeedAdapter.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ITupleTrackingFeedAdapter.java
deleted file mode 100644
index 4067508..0000000
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ITupleTrackingFeedAdapter.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.feeds.api;
-
-public interface ITupleTrackingFeedAdapter extends IDataSourceAdapter {
-
- public void tuplePersistedTimeCallback(long timestamp);
-}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/parse/ITupleForwarder.java b/asterix-common/src/main/java/org/apache/asterix/common/parse/ITupleForwarder.java
index 5ee065a..27f4fcb 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/parse/ITupleForwarder.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/parse/ITupleForwarder.java
@@ -32,10 +32,11 @@
public enum TupleForwardPolicy {
FRAME_FULL,
COUNTER_TIMER_EXPIRED,
- RATE_CONTROLLED
+ RATE_CONTROLLED,
+ FEED
}
- public void configure(Map<String, String> configuration);
+ public void configure(Map<String, String> configuration) throws HyracksDataException;
public void initialize(IHyracksCommonContext ctx, IFrameWriter frameWriter) throws HyracksDataException;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
new file mode 100644
index 0000000..acfb9d5
--- /dev/null
+++ b/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.io.File;
+
+import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+
+public class StoragePathUtil {
+ public static final String PARTITION_DIR_PREFIX = "partition_";
+ public static final String TEMP_DATASETS_STORAGE_FOLDER = "temp";
+ public static final String DATASET_INDEX_NAME_SEPARATOR = "_idx_";
+ public static final String ADAPTER_INSTANCE_PREFIX = "adapter_";
+
+ public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraints(
+ FileSplit[] splits) {
+ IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
+ String[] loc = new String[splits.length];
+ for (int p = 0; p < splits.length; p++) {
+ loc[p] = splits[p].getNodeName();
+ }
+ AlgebricksPartitionConstraint pc = new AlgebricksAbsolutePartitionConstraint(loc);
+ return new Pair<IFileSplitProvider, AlgebricksPartitionConstraint>(splitProvider, pc);
+ }
+
+ public static FileSplit getFileSplitForClusterPartition(ClusterPartition partition, File relativeFile) {
+ return new FileSplit(partition.getActiveNodeId(), new FileReference(relativeFile), partition.getIODeviceNum(),
+ partition.getPartitionId());
+ }
+
+ public static String prepareStoragePartitionPath(String storageDirName, int partitonId) {
+ return storageDirName + File.separator + StoragePathUtil.PARTITION_DIR_PREFIX + partitonId;
+ }
+
+ public static String prepareDataverseIndexName(String dataverseName, String datasetName, String idxName) {
+ return dataverseName + File.separator + datasetName + StoragePathUtil.DATASET_INDEX_NAME_SEPARATOR + idxName;
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index 2e7158d..e957ac6 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -21,10 +21,10 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IDataParserFactory;
+import org.apache.asterix.external.api.IDataSourceAdapter;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.IIndexibleExternalDataSource;
import org.apache.asterix.external.api.IIndexingAdapterFactory;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
index 9539278..bf2db9a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
@@ -21,7 +21,6 @@
import java.io.Serializable;
import java.util.Map;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterRuntimeManager.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterRuntimeManager.java
index 2eb6caa..252b43b 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterRuntimeManager.java
@@ -16,9 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.api;
-import org.apache.asterix.common.feeds.FeedId;
+import org.apache.asterix.external.feed.api.IIntakeProgressTracker;
+import org.apache.asterix.external.feed.management.FeedId;
public interface IAdapterRuntimeManager {
@@ -63,7 +64,7 @@
/**
* @return the instance of the feed adapter (an implementation of {@code IFeedAdapter}) in use.
*/
- public IDataSourceAdapter getFeedAdapter();
+ public IFeedAdapter getFeedAdapter();
/**
* @return state associated with the AdapterRuntimeManager. See {@code State}.
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java
index f5f47ec..e4435a1 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.external.api;
-import java.io.IOException;
import java.util.Map;
import org.apache.asterix.common.parse.ITupleForwarder;
@@ -39,11 +38,21 @@
* 3. setTupleForwarder(forwarder)
* 4. configure(configuration,ctx)
* 5. start(writer)
+ *
+ * pause(), resume(), and stop() are only used with feeds
+ * pause is called after start when a feed is running and the system is overwhelmed with data.
+ * resume is called after the load goes down and we are ready to receive more data.
+ * stop is called to disconnect the feed. once stop is called, no other method is called.
+ *
*/
public void start(IFrameWriter writer) throws HyracksDataException;
- public boolean stop();
+ public boolean stop() throws HyracksDataException;
+
+ public boolean pause() throws HyracksDataException;
+
+ public boolean resume() throws HyracksDataException;
public boolean handleException(Throwable th);
@@ -51,5 +60,5 @@
public void setTupleForwarder(ITupleForwarder forwarder);
- public void configure(Map<String, String> configuration, IHyracksTaskContext ctx) throws IOException;
+ public void configure(Map<String, String> configuration, IHyracksTaskContext ctx) throws HyracksDataException;
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDataSourceAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataSourceAdapter.java
similarity index 84%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDataSourceAdapter.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataSourceAdapter.java
index 9dd4e76..e37f2b1 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDataSourceAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataSourceAdapter.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.api;
import java.io.Serializable;
@@ -30,9 +30,13 @@
*/
public interface IDataSourceAdapter extends Serializable {
+ public enum AdapterType {
+ INTERNAL,
+ EXTERNAL
+ }
+
/**
* Triggers the adapter to begin ingesting data from the external source.
- *
* @param partition
* The adapter could be running with a degree of parallelism.
* partition corresponds to the i'th parallel instance.
@@ -44,17 +48,4 @@
* @throws Exception
*/
public void start(int partition, IFrameWriter writer) throws Exception;
-
- /**
- * Discontinue the ingestion of data.
- *
- * @throws Exception
- */
- public boolean stop() throws Exception;
-
- /**
- * @param e
- * @return true if the ingestion should continue post the exception else false
- */
- public boolean handleException(Throwable e);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFeedAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFeedAdapter.java
new file mode 100644
index 0000000..3261556b
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFeedAdapter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IFeedAdapter extends IDataSourceAdapter {
+ /**
+ * Pause the ingestion of data.
+ * @throws HyracksDataException
+ * @throws Exception
+ */
+ public boolean pause() throws HyracksDataException;
+
+ /**
+ * Resume the ingestion of data.
+ * @throws HyracksDataException
+ * @throws Exception
+ */
+ public boolean resume() throws HyracksDataException;
+
+ /**
+ * Discontinue the ingestion of data.
+ * @throws Exception
+ */
+ public boolean stop() throws Exception;
+
+ /**
+ * @param e
+ * @return true if the ingestion should continue post the exception else false
+ * @throws Exception
+ */
+ public boolean handleException(Throwable e);
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IStreamDataParser.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IStreamDataParser.java
index 31d6317..531d050 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IStreamDataParser.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IStreamDataParser.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.api;
import java.io.DataOutput;
+import java.io.IOException;
import java.io.InputStream;
public interface IStreamDataParser extends IDataParser {
@@ -30,10 +31,17 @@
/**
* Parse data into output AsterixDataModel binary records.
* Used with parsers that support stream sources
- *
* @param out
* DataOutput instance that for writing the parser output.
*/
-
public boolean parse(DataOutput out) throws Exception;
+
+ /**
+ * reset the parser state. this is called when a failure takes place
+ * and the job needs to continue and to do that, the parser need to
+ * be in a consistent state
+ * @return true if reset was successful, false, otherwise
+ * @throws IOException
+ */
+ public boolean reset(InputStream in) throws IOException;
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractDataFlowController.java
index d06161e..c5c8e48 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractDataFlowController.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.external.dataflow;
-import java.io.IOException;
import java.util.Map;
import org.apache.asterix.common.parse.ITupleForwarder;
@@ -48,7 +47,7 @@
}
@Override
- public void configure(Map<String, String> configuration, IHyracksTaskContext ctx) throws IOException {
+ public void configure(Map<String, String> configuration, IHyracksTaskContext ctx) {
this.configuration = configuration;
this.ctx = ctx;
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
new file mode 100644
index 0000000..aab4bf6
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.dataflow;
+
+import java.util.Map;
+
+import org.apache.asterix.common.parse.ITupleForwarder;
+import org.apache.asterix.external.api.IDataFlowController;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+
+public abstract class AbstractFeedDataFlowController implements IDataFlowController {
+ protected FeedTupleForwarder tupleForwarder;
+ protected IHyracksTaskContext ctx;
+ protected Map<String, String> configuration;
+ protected static final int NUMBER_OF_TUPLE_FIELDS = 1;
+ protected ArrayTupleBuilder tb = new ArrayTupleBuilder(NUMBER_OF_TUPLE_FIELDS);
+
+ @Override
+ public ITupleForwarder getTupleForwarder() {
+ return tupleForwarder;
+ }
+
+ @Override
+ public void setTupleForwarder(ITupleForwarder tupleForwarder) {
+ this.tupleForwarder = (FeedTupleForwarder) tupleForwarder;
+ }
+
+ protected void initializeTupleForwarder(IFrameWriter writer) throws HyracksDataException {
+ tupleForwarder.initialize(ctx, writer);
+ }
+
+ @Override
+ public void configure(Map<String, String> configuration, IHyracksTaskContext ctx) {
+ this.configuration = configuration;
+ this.ctx = ctx;
+ }
+
+ @Override
+ public boolean pause() {
+ tupleForwarder.pause();
+ return true;
+ }
+
+ @Override
+ public boolean resume() {
+ tupleForwarder.resume();
+ return true;
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
new file mode 100644
index 0000000..fe4557d
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.dataflow;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.asterix.external.api.IRawRecord;
+import org.apache.asterix.external.api.IRecordDataParser;
+import org.apache.asterix.external.api.IRecordFlowController;
+import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.util.ExternalDataExceptionUtils;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowController
+ implements IRecordFlowController<T> {
+ protected IRecordDataParser<T> dataParser;
+ protected IRecordReader<? extends T> recordReader;
+ protected long interval;
+ protected AtomicBoolean closed = new AtomicBoolean(false);
+
+ @Override
+ public void start(IFrameWriter writer) throws HyracksDataException {
+ HyracksDataException hde = null;
+ try {
+ initializeTupleForwarder(writer);
+ while (recordReader.hasNext()) {
+ IRawRecord<? extends T> record = recordReader.next();
+ if (record == null) {
+ Thread.sleep(interval);
+ continue;
+ }
+ tb.reset();
+ dataParser.parse(record, tb.getDataOutput());
+ tb.addFieldEndOffset();
+ tupleForwarder.addTuple(tb);
+ }
+ } catch (Throwable th) {
+ hde = new HyracksDataException(th);
+ }
+ try {
+ tupleForwarder.close();
+ } catch (Throwable th) {
+ hde = ExternalDataExceptionUtils.suppress(hde, th);
+ }
+ try {
+ recordReader.close();
+ } catch (Throwable th) {
+ hde = ExternalDataExceptionUtils.suppress(hde, th);
+ throw hde;
+ } finally {
+ closeSignal();
+ }
+ }
+
+ private void closeSignal() {
+ synchronized (closed) {
+ closed.set(true);
+ closed.notifyAll();
+ }
+ }
+
+ private void waitForSignal() throws InterruptedException {
+ synchronized (closed) {
+ while (!closed.get()) {
+ closed.wait();
+ }
+ }
+ }
+
+ @Override
+ public boolean stop() throws HyracksDataException {
+ if (recordReader.stop()) {
+ try {
+ waitForSignal();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return true;
+ }
+
+ @Override
+ public void setRecordParser(IRecordDataParser<T> dataParser) {
+ this.dataParser = dataParser;
+ }
+
+ @Override
+ public void setRecordReader(IRecordReader<T> recordReader) {
+ this.recordReader = recordReader;
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
new file mode 100644
index 0000000..4ef5f6d
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.dataflow;
+
+import org.apache.asterix.external.api.IStreamDataParser;
+import org.apache.asterix.external.api.IStreamFlowController;
+import org.apache.asterix.external.input.stream.AInputStream;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FeedStreamDataFlowController extends AbstractFeedDataFlowController implements IStreamFlowController {
+
+ private IStreamDataParser dataParser;
+ private AInputStream stream;
+
+ @Override
+ public void start(IFrameWriter writer) throws HyracksDataException {
+ try {
+ initializeTupleForwarder(writer);
+ while (true) {
+ tb.reset();
+ if (!dataParser.parse(tb.getDataOutput())) {
+ break;
+ }
+ tb.addFieldEndOffset();
+ tupleForwarder.addTuple(tb);
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ } finally {
+ tupleForwarder.close();
+ }
+ }
+
+ @Override
+ public boolean stop() throws HyracksDataException {
+ try {
+ if (stream.stop()) {
+ return true;
+ }
+ stream.close();
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ boolean handled = true;
+ try {
+ handled &= stream.skipError();
+ if (handled) {
+ handled &= dataParser.reset(stream);
+ }
+ } catch (Exception e) {
+ th.addSuppressed(e);
+ return false;
+ }
+ return handled;
+ }
+
+ @Override
+ public void setStreamParser(IStreamDataParser dataParser) {
+ this.dataParser = dataParser;
+ }
+
+ public void setStream(AInputStream stream) {
+ this.stream = stream;
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
new file mode 100644
index 0000000..d1707665
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.dataflow;
+
+import java.util.Map;
+
+import org.apache.asterix.common.parse.ITupleForwarder;
+import org.apache.asterix.external.util.DataflowUtils;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+
+public class FeedTupleForwarder implements ITupleForwarder {
+
+ private FrameTupleAppender appender;
+ private IFrame frame;
+ private IFrameWriter writer;
+ private boolean paused = false;
+
+ @Override
+ public void configure(Map<String, String> configuration) {
+ }
+
+ @Override
+ public void initialize(IHyracksCommonContext ctx, IFrameWriter writer) throws HyracksDataException {
+ this.frame = new VSizeFrame(ctx);
+ this.writer = writer;
+ this.appender = new FrameTupleAppender(frame);
+ }
+
+ @Override
+ public void addTuple(ArrayTupleBuilder tb) throws HyracksDataException {
+ if (paused) {
+ synchronized (this) {
+ while (paused) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+ }
+ DataflowUtils.addTupleToFrame(appender, tb, writer);
+ }
+
+ public void pause() {
+ paused = true;
+ }
+
+ public synchronized void resume() {
+ paused = false;
+ notifyAll();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (appender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(frame.getBuffer(), writer);
+ }
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
index ad8e791..9353a40 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
@@ -57,7 +57,7 @@
@Override
public boolean stop() {
- return false;
+ return recordReader.stop();
}
@Override
@@ -74,4 +74,14 @@
public void setRecordReader(IRecordReader<T> recordReader) throws Exception {
this.recordReader = recordReader;
}
+
+ @Override
+ public boolean pause() throws HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public boolean resume() throws HyracksDataException {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/StreamDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/StreamDataFlowController.java
index 3016470..43738eb 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/StreamDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/StreamDataFlowController.java
@@ -61,4 +61,14 @@
public void setStreamParser(IStreamDataParser dataParser) {
this.dataParser = dataParser;
}
+
+ @Override
+ public boolean pause() throws HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public boolean resume() throws HyracksDataException {
+ return false;
+ }
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterIdentifier.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/AdapterIdentifier.java
similarity index 75%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterIdentifier.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/AdapterIdentifier.java
index f7e528b..6c02500 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterIdentifier.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/AdapterIdentifier.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.dataset.adapter;
import java.io.Serializable;
@@ -27,25 +27,25 @@
private static final long serialVersionUID = 1L;
- private final String namespace;
- private final String name;
+ private final String dataverseName;
+ private final String adapterName;
public AdapterIdentifier(String namespace, String name) {
- this.namespace = namespace;
- this.name = name;
+ this.dataverseName = namespace;
+ this.adapterName = name;
}
public String getNamespace() {
- return namespace;
+ return dataverseName;
}
public String getName() {
- return name;
+ return adapterName;
}
@Override
public int hashCode() {
- return (namespace + "@" + name).hashCode();
+ return (dataverseName + "@" + adapterName).hashCode();
}
@@ -60,7 +60,7 @@
if (!(o instanceof AdapterIdentifier)) {
return false;
}
- return namespace.equals(((AdapterIdentifier) o).getNamespace())
- && name.equals(((AdapterIdentifier) o).getName());
+ AdapterIdentifier a = (AdapterIdentifier) o;
+ return dataverseName.equals(a.getNamespace()) && adapterName.equals(a.getName());
}
}
\ No newline at end of file
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/GenericAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/GenericAdapter.java
index 74e98dd..d19eedf 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/GenericAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/GenericAdapter.java
@@ -18,11 +18,12 @@
*/
package org.apache.asterix.external.dataset.adapter;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
import org.apache.asterix.external.api.IDataFlowController;
+import org.apache.asterix.external.api.IFeedAdapter;
import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
-public class GenericAdapter implements IDataSourceAdapter {
+public class GenericAdapter implements IFeedAdapter {
private static final long serialVersionUID = 1L;
private final IDataFlowController controller;
@@ -32,12 +33,12 @@
}
@Override
- public void start(int partition, IFrameWriter writer) throws Exception {
+ public void start(int partition, IFrameWriter writer) throws HyracksDataException {
controller.start(writer);
}
@Override
- public boolean stop() throws Exception {
+ public boolean stop() throws HyracksDataException {
return controller.stop();
}
@@ -45,4 +46,14 @@
public boolean handleException(Throwable e) {
return controller.handleException(e);
}
+
+ @Override
+ public boolean pause() throws HyracksDataException {
+ return controller.pause();
+ }
+
+ @Override
+ public boolean resume() throws HyracksDataException {
+ return controller.resume();
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/StreamBasedAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/StreamBasedAdapter.java
deleted file mode 100644
index 3f10dc4..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/StreamBasedAdapter.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.dataset.adapter;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.std.file.ITupleParser;
-import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
-
-public abstract class StreamBasedAdapter implements IDataSourceAdapter {
-
- private static final long serialVersionUID = 1L;
-
- protected static final Logger LOGGER = Logger.getLogger(StreamBasedAdapter.class.getName());
-
- public abstract InputStream getInputStream(int partition) throws IOException;
-
- protected final ITupleParser tupleParser;
-
- protected final IAType sourceDatatype;
-
- public StreamBasedAdapter(ITupleParserFactory parserFactory, IAType sourceDatatype, IHyracksTaskContext ctx,
- int partition) throws HyracksDataException {
- this.tupleParser = parserFactory.createTupleParser(ctx);
- this.sourceDatatype = sourceDatatype;
- }
-
- @Override
- public void start(int partition, IFrameWriter writer) throws Exception {
- InputStream in = getInputStream(partition);
- if (in != null) {
- tupleParser.parse(in, writer);
- } else {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning(
- "Could not obtain input stream for parsing from adapter " + this + "[" + partition + "]");
- }
- }
- }
-
-}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ICentralFeedManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ICentralFeedManager.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ICentralFeedManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ICentralFeedManager.java
index 0092e6b..4f0ed77 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ICentralFeedManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ICentralFeedManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.io.IOException;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IExceptionHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IExceptionHandler.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IExceptionHandler.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IExceptionHandler.java
index db2c890..ec0af1c 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IExceptionHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IExceptionHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.nio.ByteBuffer;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeed.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeed.java
new file mode 100644
index 0000000..6865522
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeed.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.feed.api;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.external.feed.management.FeedId;
+
+public interface IFeed extends Serializable {
+
+ public enum FeedType {
+ /**
+ * A feed that derives its data from an external source.
+ */
+ PRIMARY,
+
+ /**
+ * A feed that derives its data from another primary or secondary feed.
+ */
+ SECONDARY
+ }
+
+ public FeedType getFeedType();
+
+ public FunctionSignature getAppliedFunction();
+
+ public String getFeedName();
+
+ public String getDataverseName();
+
+ public FeedId getFeedId();
+
+ public Map<String, String> getAdapterConfiguration();
+
+ public String getAdapterName();
+
+}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedConnectionManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedConnectionManager.java
similarity index 86%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedConnectionManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedConnectionManager.java
index 014a868..35d4cd7 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedConnectionManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedConnectionManager.java
@@ -16,15 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.io.IOException;
import java.util.List;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedRuntime;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeManager;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedRuntimeManager;
+import org.apache.asterix.external.feed.runtime.FeedRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
/**
* Handle (de)registration of feeds for delivery of control messages.
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedFrameHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
similarity index 91%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedFrameHandler.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
index 3a95e51..9cb0feb 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedFrameHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
@@ -16,12 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.nio.ByteBuffer;
import java.util.Iterator;
-import org.apache.asterix.common.feeds.DataBucket;
+import org.apache.asterix.external.feed.dataflow.DataBucket;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IFeedFrameHandler {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedJoint.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
similarity index 89%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedJoint.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
index 59d807a..d990e45 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedJoint.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
@@ -16,15 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.util.List;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedJointKey;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedJointKey;
public interface IFeedJoint {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleEventSubscriber.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleEventSubscriber.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleEventSubscriber.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleEventSubscriber.java
index 94af74b..0c8724e 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleEventSubscriber.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleEventSubscriber.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import org.apache.asterix.common.exceptions.AsterixException;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleIntakeEventSubscriber.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleIntakeEventSubscriber.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleIntakeEventSubscriber.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleIntakeEventSubscriber.java
index bff2589..b9caa0d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleIntakeEventSubscriber.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleIntakeEventSubscriber.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedIntakeInfo;
+import org.apache.asterix.external.feed.watch.FeedIntakeInfo;
public interface IFeedLifecycleIntakeEventSubscriber extends IFeedLifecycleEventSubscriber {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleListener.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
similarity index 88%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleListener.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
index 0ae5f56..ce82aaf 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLifecycleListener.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
@@ -16,14 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.util.List;
import org.apache.asterix.common.api.IClusterEventsSubscriber;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedJointKey;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedJointKey;
import org.apache.hyracks.api.job.IJobLifecycleListener;
public interface IFeedLifecycleListener extends IJobLifecycleListener, IClusterEventsSubscriber {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLoadManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLoadManager.java
similarity index 75%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLoadManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLoadManager.java
index 7baa229..f511979 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedLoadManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLoadManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.util.Collection;
import java.util.List;
@@ -24,14 +24,14 @@
import org.json.JSONException;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedActivity;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.NodeLoadReport;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.message.FeedCongestionMessage;
-import org.apache.asterix.common.feeds.message.FeedReportMessage;
-import org.apache.asterix.common.feeds.message.ScaleInReportMessage;
-import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.FeedCongestionMessage;
+import org.apache.asterix.external.feed.message.FeedReportMessage;
+import org.apache.asterix.external.feed.message.ScaleInReportMessage;
+import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.watch.FeedActivity;
+import org.apache.asterix.external.feed.watch.NodeLoadReport;
public interface IFeedLoadManager {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedManager.java
similarity index 75%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedManager.java
index 768b11f..b3ad0a5 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedManager.java
@@ -16,15 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import org.apache.asterix.common.config.AsterixFeedProperties;
-import org.apache.asterix.common.feeds.api.IFeedConnectionManager;
-import org.apache.asterix.common.feeds.api.IFeedMemoryManager;
-import org.apache.asterix.common.feeds.api.IFeedMessageService;
-import org.apache.asterix.common.feeds.api.IFeedMetadataManager;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
-import org.apache.asterix.common.feeds.api.IFeedSubscriptionManager;
/**
* Provides access to services related to feed management within a node controller
@@ -32,15 +26,7 @@
public interface IFeedManager {
/**
- * gets the nodeId associated with the host node controller
- *
- * @return the nodeId associated with the host node controller
- */
- public String getNodeId();
-
- /**
* gets the handle to the singleton instance of subscription manager
- *
* @return the singleton instance of subscription manager
* @see IFeedSubscriptionManager
*/
@@ -48,7 +34,6 @@
/**
* gets the handle to the singleton instance of connection manager
- *
* @return the singleton instance of connection manager
* @see IFeedConnectionManager
*/
@@ -56,7 +41,6 @@
/**
* gets the handle to the singleton instance of memory manager
- *
* @return the singleton instance of memory manager
* @see IFeedMemoryManager
*/
@@ -64,7 +48,6 @@
/**
* gets the handle to the singleton instance of feed metadata manager
- *
* @return the singleton instance of feed metadata manager
* @see IFeedMetadataManager
*/
@@ -72,7 +55,6 @@
/**
* gets the handle to the singleton instance of feed metric collector
- *
* @return the singleton instance of feed metric collector
* @see IFeedMetricCollector
*/
@@ -80,18 +62,11 @@
/**
* gets the handle to the singleton instance of feed message service
- *
* @return the singleton instance of feed message service
* @see IFeedMessageService
*/
public IFeedMessageService getFeedMessageService();
- /**
- * gets the asterix configuration
- *
- * @return asterix configuration
- * @see AsterixFeedProperties
- */
public AsterixFeedProperties getAsterixFeedProperties();
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMemoryComponent.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMemoryComponent.java
similarity index 97%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMemoryComponent.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMemoryComponent.java
index 817b750..313dc1b 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMemoryComponent.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMemoryComponent.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
/**
* Represents an in-memory components required for storing frames that contain feed tuples.
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMemoryManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMemoryManager.java
similarity index 93%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMemoryManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMemoryManager.java
index 4902606..fb81373 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMemoryManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMemoryManager.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.api.IFeedMemoryComponent.Type;
+import org.apache.asterix.external.feed.api.IFeedMemoryComponent.Type;
/**
* Provides management of memory allocated for handling feed data flow through the node controller
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMessage.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMessage.java
index 14b8e0a..aa1af3a 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMessage.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.io.Serializable;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMessageService.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMessageService.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMessageService.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMessageService.java
index 12f53be..42f71a7 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMessageService.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMessageService.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
/**
* Provides the functionality of sending a meesage ({@code IFeedMessage} to the {@code CentralFeedManager}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMetadataManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMetadataManager.java
similarity index 92%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMetadataManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMetadataManager.java
index 127b97c..3712678 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMetadataManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMetadataManager.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
public interface IFeedMetadataManager {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMetricCollector.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMetricCollector.java
similarity index 88%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMetricCollector.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMetricCollector.java
index b78d81e..c35587c 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedMetricCollector.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedMetricCollector.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
public interface IFeedMetricCollector {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedOperatorOutputSideHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedOperatorOutputSideHandler.java
similarity index 69%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedOperatorOutputSideHandler.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedOperatorOutputSideHandler.java
index 5dec7e3..a8d0552 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedOperatorOutputSideHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedOperatorOutputSideHandler.java
@@ -16,11 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.FeedId;
+import org.apache.asterix.external.feed.management.FeedId;
import org.apache.hyracks.api.comm.IFrameWriter;
+/**
+ * Provides for output-side buffering for a feed runtime.
+ * Right now, we only have a single output side handler
+ * and we can probably remove it completely.
+ * ______
+ * | |
+ * ============|core |============
+ * ============| op |============
+ * |______|^^^^^^^^^^^^
+ * Output Side
+ * Handler
+ *
+ **/
public interface IFeedOperatorOutputSideHandler extends IFrameWriter {
public enum Type {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedProvider.java
similarity index 89%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedProvider.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedProvider.java
index 7565004..9eced07 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedProvider.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.FeedId;
+import org.apache.asterix.external.feed.management.FeedId;
public interface IFeedProvider {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedRuntime.java
similarity index 89%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedRuntime.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedRuntime.java
index 4fcd631..269725d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedRuntime.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.comm.IFrameWriter;
public interface IFeedRuntime {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedService.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedService.java
similarity index 94%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedService.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedService.java
index ac51c95..3d3e0e5 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedService.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedService.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
public interface IFeedService {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedSubscriptionManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedSubscriptionManager.java
similarity index 91%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedSubscriptionManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedSubscriptionManager.java
index 91ac4c4..ec4c396 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedSubscriptionManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedSubscriptionManager.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
public interface IFeedSubscriptionManager {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedTrackingManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedTrackingManager.java
similarity index 83%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedTrackingManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedTrackingManager.java
index 280c4d9..6576e09 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedTrackingManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedTrackingManager.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedTupleCommitAckMessage;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
public interface IFeedTrackingManager {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWork.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWork.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWork.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWork.java
index 06fcb76..2f5379b 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWork.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWork.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
/**
* Represents a feed management task. The task is executed asynchronously.
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWorkEventListener.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWorkEventListener.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWorkEventListener.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWorkEventListener.java
index 59b46d1..e5797d0 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWorkEventListener.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWorkEventListener.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
/**
* Provides a callback mechanism that in invoked for events related to
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWorkManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWorkManager.java
similarity index 94%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWorkManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWorkManager.java
index 31506ca..37661b7 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedWorkManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedWorkManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
public interface IFeedWorkManager {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFrameEventCallback.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFrameEventCallback.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFrameEventCallback.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFrameEventCallback.java
index 13a0af0..647d847 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFrameEventCallback.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFrameEventCallback.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
public interface IFrameEventCallback {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePostProcessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFramePostProcessor.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePostProcessor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFramePostProcessor.java
index ed74037..eab7a64 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePostProcessor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFramePostProcessor.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.nio.ByteBuffer;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePreprocessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFramePreprocessor.java
similarity index 94%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePreprocessor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFramePreprocessor.java
index 59a6c97..55461b7 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePreprocessor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFramePreprocessor.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.nio.ByteBuffer;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IIntakeProgressTracker.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IIntakeProgressTracker.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IIntakeProgressTracker.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IIntakeProgressTracker.java
index f2c9f63..4848ed8 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IIntakeProgressTracker.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IIntakeProgressTracker.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.util.Map;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IMessageReceiver.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IMessageReceiver.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
index bc86e86..bdfbfdb 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IMessageReceiver.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
public interface IMessageReceiver<T> {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscribableRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscribableRuntime.java
similarity index 86%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscribableRuntime.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscribableRuntime.java
index 14b89f7..ee07188 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscribableRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscribableRuntime.java
@@ -16,13 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.util.List;
-import org.apache.asterix.common.feeds.CollectionRuntime;
-import org.apache.asterix.common.feeds.DistributeFeedFrameWriter;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.CollectionRuntime;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
/**
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscriberRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriberRuntime.java
similarity index 89%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscriberRuntime.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriberRuntime.java
index 3b2157f..4d3e607 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscriberRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriberRuntime.java
@@ -16,11 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
import java.util.Map;
-import org.apache.asterix.common.feeds.FeedFrameCollector;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
public interface ISubscriberRuntime {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscriptionProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriptionProvider.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscriptionProvider.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriptionProvider.java
index b9264b8..b94a52e 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/ISubscriptionProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriptionProvider.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.api;
+package org.apache.asterix.external.feed.api;
-import org.apache.asterix.common.feeds.FeedId;
+import org.apache.asterix.external.feed.management.FeedId;
public interface ISubscriptionProvider {
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/CollectTransformFeedFrameWriter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/CollectTransformFeedFrameWriter.java
similarity index 77%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/CollectTransformFeedFrameWriter.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/CollectTransformFeedFrameWriter.java
index 30b369a..18b6ec0 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/CollectTransformFeedFrameWriter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/CollectTransformFeedFrameWriter.java
@@ -16,14 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.api.IFeedOperatorOutputSideHandler;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.IFeedOperatorOutputSideHandler;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.comm.VSizeFrame;
@@ -35,26 +35,25 @@
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+// Simply a delivery frame writer. I think we can simply get rid of this at some point {TODO}.
public class CollectTransformFeedFrameWriter implements IFeedOperatorOutputSideHandler {
- private final FeedConnectionId connectionId;
- private IFrameWriter downstreamWriter;
- private final FrameTupleAccessor inputFrameTupleAccessor;
- private final FrameTupleAppender tupleAppender;
- private final IFrame frame;
+ private final FeedConnectionId connectionId; // [Dataverse - Feed - Dataset]
+ private IFrameWriter downstreamWriter; // Writer to next (Operator/Connector)
+ private final FrameTupleAccessor inputFrameTupleAccessor; // Accessing input frame tuples
+ private final FrameTupleAppender tupleAppender; // Append tuples to output frame
+ private final IFrame frame; // Output frame
private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(1);
public CollectTransformFeedFrameWriter(IHyracksTaskContext ctx, IFrameWriter downstreamWriter,
ISubscribableRuntime sourceRuntime, RecordDescriptor outputRecordDescriptor, FeedConnectionId connectionId)
- throws HyracksDataException {
- this.downstreamWriter = downstreamWriter;
- RecordDescriptor inputRecordDescriptor = sourceRuntime.getRecordDescriptor();
- inputFrameTupleAccessor = new FrameTupleAccessor(inputRecordDescriptor);
- tupleAppender = new FrameTupleAppender();
- frame = new VSizeFrame(ctx);
- tupleAppender.reset(frame, true);
+ throws HyracksDataException {
this.connectionId = connectionId;
+ this.downstreamWriter = downstreamWriter;
+ inputFrameTupleAccessor = new FrameTupleAccessor(sourceRuntime.getRecordDescriptor());
+ frame = new VSizeFrame(ctx);
+ tupleAppender = new FrameTupleAppender(frame);
}
@Override
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/DataBucket.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DataBucket.java
similarity index 81%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/DataBucket.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DataBucket.java
index ccd6547..3943ced 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/DataBucket.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DataBucket.java
@@ -16,23 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicInteger;
+/**
+ * A {@DataBucket} is a wrapper around {@ByteBuffer} that expects certain number of receipients
+ */
public class DataBucket {
private static final AtomicInteger globalBucketId = new AtomicInteger(0);
- private final ByteBuffer content;
- private final AtomicInteger readCount;
- private final int bucketId;
-
- private int desiredReadCount;
- private ContentType contentType;
-
- private final DataBucketPool pool;
+ private final ByteBuffer content; // Content
+ private final AtomicInteger readCount; // How many reads?
+ private final int bucketId; // Id
+ private int desiredReadCount; // Number of expected readers
+ private ContentType contentType; // Data, End of stream, or End of spilled data
+ private final DataBucketPool pool; // Pool of buckets
public enum ContentType {
DATA, // data (feed tuple)
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/DataBucketPool.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DataBucketPool.java
similarity index 93%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/DataBucketPool.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DataBucketPool.java
index 2e7e60c..339469e 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/DataBucketPool.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DataBucketPool.java
@@ -16,12 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.util.Stack;
-import org.apache.asterix.common.feeds.api.IFeedMemoryComponent;
-import org.apache.asterix.common.feeds.api.IFeedMemoryManager;
+import org.apache.asterix.external.feed.api.IFeedMemoryComponent;
+import org.apache.asterix.external.feed.api.IFeedMemoryManager;
/**
* Represents a pool of reusable {@link DataBucket}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/DistributeFeedFrameWriter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
similarity index 83%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/DistributeFeedFrameWriter.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
index d0e371e..7367d5a 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/DistributeFeedFrameWriter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -24,10 +24,13 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedOperatorOutputSideHandler;
-import org.apache.asterix.common.feeds.api.IFeedOperatorOutputSideHandler.Type;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedOperatorOutputSideHandler;
+import org.apache.asterix.external.feed.api.IFeedOperatorOutputSideHandler.Type;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -46,7 +49,10 @@
/** A unique identifier for the feed to which the incoming tuples belong. **/
private final FeedId feedId;
- /** An instance of FrameDistributor that provides the mechanism for distributing a frame to multiple readers, each operating in isolation. **/
+ /**
+ * An instance of FrameDistributor that provides the mechanism for distributing a frame to multiple readers, each
+ * operating in isolation.
+ **/
private final FrameDistributor frameDistributor;
/** The original frame writer instantiated as part of job creation **/
@@ -69,6 +75,16 @@
this.writer = writer;
}
+ /**
+ * @param fpa
+ * Feed policy accessor
+ * @param frameWriter
+ * the writer which will deliver the buffers
+ * @param connectionId
+ * (Dataverse - Dataset - Feed)
+ * @return A frame collector.
+ * @throws Exception
+ */
public FeedFrameCollector subscribeFeed(FeedPolicyAccessor fpa, IFrameWriter frameWriter,
FeedConnectionId connectionId) throws Exception {
FeedFrameCollector collector = null;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedCollectRuntimeInputHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
similarity index 88%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedCollectRuntimeInputHandler.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
index 97dc4f8..6761d9a 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedCollectRuntimeInputHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
@@ -16,12 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.asterix.common.feeds.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedExceptionHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedExceptionHandler.java
similarity index 66%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedExceptionHandler.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedExceptionHandler.java
index f1728ce..483ba19 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedExceptionHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedExceptionHandler.java
@@ -16,50 +16,44 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
-import java.io.DataInputStream;
import java.nio.ByteBuffer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.FrameDataException;
-import org.apache.asterix.common.feeds.api.IExceptionHandler;
-import org.apache.asterix.common.feeds.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IExceptionHandler;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.util.FeedFrameUtil;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
public class FeedExceptionHandler implements IExceptionHandler {
private static final Logger LOGGER = Logger.getLogger(FeedExceptionHandler.class.getName());
+ //TODO: Enable logging
private final IHyracksTaskContext ctx;
private final FrameTupleAccessor fta;
- private final RecordDescriptor recordDesc;
- private final IFeedManager feedManager;
- private final FeedConnectionId connectionId;
public FeedExceptionHandler(IHyracksTaskContext ctx, FrameTupleAccessor fta, RecordDescriptor recordDesc,
IFeedManager feedManager, FeedConnectionId connectionId) {
this.ctx = ctx;
this.fta = fta;
- this.recordDesc = recordDesc;
- this.feedManager = feedManager;
- this.connectionId = connectionId;
}
+ @Override
public ByteBuffer handleException(Exception e, ByteBuffer frame) {
try {
if (e instanceof FrameDataException) {
fta.reset(frame);
FrameDataException fde = (FrameDataException) e;
int tupleIndex = fde.getTupleIndex();
-
- // logging
try {
logExceptionCausingTuple(tupleIndex, e);
} catch (Exception ex) {
@@ -68,8 +62,7 @@
LOGGER.warning("Unable to log exception causing tuple due to..." + ex.getMessage());
}
}
- // slicing
- return FeedFrameUtil.getSlicedFrame(ctx, tupleIndex, fta);
+ return FeedFrameUtil.removeBadTuple(ctx, tupleIndex, fta);
} else {
return null;
}
@@ -83,26 +76,5 @@
}
private void logExceptionCausingTuple(int tupleIndex, Exception e) throws HyracksDataException, AsterixException {
-
- ByteBufferInputStream bbis = new ByteBufferInputStream();
- DataInputStream di = new DataInputStream(bbis);
-
- int start = fta.getTupleStartOffset(tupleIndex) + fta.getFieldSlotsLength();
- bbis.setByteBuffer(fta.getBuffer(), start);
-
- Object[] record = new Object[recordDesc.getFieldCount()];
-
- for (int i = 0; i < record.length; ++i) {
- Object instance = recordDesc.getFields()[i].deserialize(di);
- if (i == 0) {
- String tuple = String.valueOf(instance);
- feedManager.getFeedMetadataManager().logTuple(connectionId, tuple, e.getMessage(), feedManager);
- } else {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning(", " + String.valueOf(instance));
- }
- }
- }
-
}
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameCache.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameCache.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
index 55a7fb8..cd040c9 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameCache.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -25,7 +25,8 @@
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.asterix.common.feeds.FeedConstants.StatisticsConstants;
+import org.apache.asterix.external.feed.message.MessageReceiver;
+import org.apache.asterix.external.util.FeedConstants.StatisticsConstants;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.comm.VSizeFrame;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameCollector.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCollector.java
similarity index 94%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameCollector.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCollector.java
index a8c0e8f..0d53524 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameCollector.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCollector.java
@@ -16,17 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
import java.util.logging.Level;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.IMessageReceiver;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.MessageReceiver;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public class FeedFrameCollector extends MessageReceiver<DataBucket> implements IMessageReceiver<DataBucket> {
+public class FeedFrameCollector extends MessageReceiver<DataBucket> {
private final FeedConnectionId connectionId;
private final FrameDistributor frameDistributor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameDiscarder.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameDiscarder.java
similarity index 85%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameDiscarder.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameDiscarder.java
index 8609366..53ee475 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameDiscarder.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameDiscarder.java
@@ -16,13 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public class FeedFrameDiscarder {
private static final Logger LOGGER = Logger.getLogger(FeedFrameSpiller.class.getName());
@@ -35,7 +39,7 @@
private int nDiscarded;
public FeedFrameDiscarder(FeedConnectionId connectionId, FeedRuntimeId runtimeId, FeedPolicyAccessor policyAccessor,
- FeedRuntimeInputHandler inputHandler) throws IOException {
+ FeedRuntimeInputHandler inputHandler) throws HyracksDataException {
this.connectionId = connectionId;
this.runtimeId = runtimeId;
this.policyAccessor = policyAccessor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameHandlers.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameHandlers.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
index c4a2ce0..6ad00f1 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameHandlers.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -31,8 +31,10 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IFeedFrameHandler;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedFrameHandler;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.message.MessageReceiver;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameSpiller.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameSpiller.java
similarity index 72%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameSpiller.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameSpiller.java
index 86187b8..c9a29ac 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameSpiller.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameSpiller.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -32,9 +32,13 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
public class FeedFrameSpiller {
@@ -51,14 +55,14 @@
private int spilledFrameCount = 0;
public FeedFrameSpiller(IHyracksTaskContext ctx, FeedConnectionId connectionId, FeedRuntimeId runtimeId,
- FeedPolicyAccessor policyAccessor) throws IOException {
+ FeedPolicyAccessor policyAccessor) throws HyracksDataException {
this.ctx = ctx;
this.connectionId = connectionId;
this.runtimeId = runtimeId;
this.policyAccessor = policyAccessor;
}
- public boolean processMessage(ByteBuffer message) throws Exception {
+ public boolean processMessage(ByteBuffer message) throws HyracksDataException {
if (!fileCreated) {
createFile();
fileCreated = true;
@@ -67,7 +71,11 @@
if (maxAllowed != FeedPolicyAccessor.NO_LIMIT && bytesWritten + message.array().length > maxAllowed) {
return false;
} else {
- bos.write(message.array());
+ try {
+ bos.write(message.array());
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
bytesWritten += message.array().length;
spilledFrameCount++;
if (LOGGER.isLoggable(Level.WARNING)) {
@@ -77,25 +85,29 @@
}
}
- private void createFile() throws IOException {
- Date date = new Date();
- String dateSuffix = date.toString().replace(' ', '_');
- String fileName = connectionId.getFeedId() + "_" + connectionId.getDatasetName() + "_"
- + runtimeId.getFeedRuntimeType() + "_" + runtimeId.getPartition() + "_" + dateSuffix;
+ private void createFile() throws HyracksDataException {
+ try {
+ Date date = new Date();
+ String dateSuffix = date.toString().replace(' ', '_');
+ String fileName = connectionId.getFeedId() + "_" + connectionId.getDatasetName() + "_"
+ + runtimeId.getFeedRuntimeType() + "_" + runtimeId.getPartition() + "_" + dateSuffix;
- file = new File(fileName);
- if (!file.exists()) {
- boolean success = file.createNewFile();
- if (!success) {
- throw new IOException("Unable to create spill file " + fileName + " for feed " + runtimeId);
- } else {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Created spill file " + file.getAbsolutePath());
+ file = new File(fileName);
+ if (!file.exists()) {
+ boolean success = file.createNewFile();
+ if (!success) {
+ throw new HyracksDataException(
+ "Unable to create spill file " + fileName + " for feed " + runtimeId);
+ } else {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Created spill file " + file.getAbsolutePath());
+ }
}
}
+ bos = new BufferedOutputStream(new FileOutputStream(file));
+ } catch (Throwable th) {
+ throw new HyracksDataException(th);
}
- bos = new BufferedOutputStream(new FileOutputStream(file));
-
}
public Iterator<ByteBuffer> replayData() throws Exception {
@@ -131,9 +143,9 @@
@Override
public ByteBuffer next() {
- IFrame frame = null;
+ IFrame frame = null;
try {
- frame = new VSizeFrame(ctx);
+ frame = new VSizeFrame(ctx);
Arrays.fill(frame.getBuffer().array(), (byte) 0);
bis.read(frame.getBuffer().array(), 0, frame.getFrameSize());
readFrameCount++;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameTupleAccessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameTupleAccessor.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameTupleAccessor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameTupleAccessor.java
index 9645bf9..f08243e 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameTupleAccessor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameTupleAccessor.java
@@ -16,11 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
-import org.apache.asterix.common.feeds.FeedConstants.StatisticsConstants;
+import org.apache.asterix.external.util.FeedConstants.StatisticsConstants;
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedFrameTupleDecorator.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameTupleDecorator.java
similarity index 96%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedFrameTupleDecorator.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameTupleDecorator.java
index 6ee14d8..d43f90d 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedFrameTupleDecorator.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameTupleDecorator.java
@@ -16,13 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.asterix.builders.IARecordBuilder;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedConstants.StatisticsConstants;
+import org.apache.asterix.external.util.FeedConstants.StatisticsConstants;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AInt64;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeInputHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
similarity index 87%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeInputHandler.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
index 6642df1..3a46b1a 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeInputHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
@@ -16,23 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.DataBucket.ContentType;
-import org.apache.asterix.common.feeds.api.IExceptionHandler;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedMemoryComponent;
-import org.apache.asterix.common.feeds.api.IFeedMessage;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.common.feeds.message.FeedCongestionMessage;
-import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.api.IExceptionHandler;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedMemoryComponent;
+import org.apache.asterix.external.feed.api.IFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.dataflow.DataBucket.ContentType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.FeedCongestionMessage;
+import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.feed.watch.MonitoredBuffer;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
@@ -41,7 +44,18 @@
/**
* Provides for error-handling and input-side buffering for a feed runtime.
- */
+ * The input handler is buffering in:
+ * 1. FeedMetaComputeNodePushable.initializeNewFeedRuntime();
+ * 2. FeedMetaStoreNodePushable.initializeNewFeedRuntime();
+ * ______
+ * | |
+ * ============|core |============
+ * ============| op |============
+ * ^^^^^^^^^^^^|______|
+ * Input Side
+ * Handler
+ *
+ **/
public class FeedRuntimeInputHandler implements IFrameWriter {
private static Logger LOGGER = Logger.getLogger(FeedRuntimeInputHandler.class.getName());
@@ -49,13 +63,12 @@
private final FeedConnectionId connectionId;
private final FeedRuntimeId runtimeId;
private final FeedPolicyAccessor feedPolicyAccessor;
- private boolean bufferingEnabled;
private final IExceptionHandler exceptionHandler;
private final FeedFrameDiscarder discarder;
private final FeedFrameSpiller spiller;
private final FeedPolicyAccessor fpa;
private final IFeedManager feedManager;
-
+ private boolean bufferingEnabled;
private IFrameWriter coreOperator;
private MonitoredBuffer mBuffer;
private DataBucketPool pool;
@@ -69,8 +82,15 @@
private FrameEventCallback frameEventCallback;
public FeedRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId, FeedRuntimeId runtimeId,
+ IFrameWriter coreOperator, FeedPolicyAccessor fpa, FrameTupleAccessor fta, RecordDescriptor recordDesc,
+ IFeedManager feedManager, int nPartitions) throws HyracksDataException {
+ this(ctx, connectionId, runtimeId, coreOperator, fpa, fpa.bufferingEnabled(), fta, recordDesc, feedManager,
+ nPartitions);
+ }
+
+ public FeedRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId, FeedRuntimeId runtimeId,
IFrameWriter coreOperator, FeedPolicyAccessor fpa, boolean bufferingEnabled, FrameTupleAccessor fta,
- RecordDescriptor recordDesc, IFeedManager feedManager, int nPartitions) throws IOException {
+ RecordDescriptor recordDesc, IFeedManager feedManager, int nPartitions) throws HyracksDataException {
this.connectionId = connectionId;
this.runtimeId = runtimeId;
this.coreOperator = coreOperator;
@@ -245,8 +265,15 @@
while (!frameProcessed) {
try {
if (!bufferingEnabled) {
- coreOperator.nextFrame(frame); // synchronous
- mBuffer.sendReport(frame);
+ if (frame == null) {
+ setFinished(true);
+ synchronized (coreOperator) {
+ coreOperator.notifyAll();
+ }
+ } else {
+ coreOperator.nextFrame(frame); // synchronous
+ mBuffer.sendReport(frame);
+ }
} else {
DataBucket bucket = pool.getDataBucket();
if (bucket != null) {
@@ -285,6 +312,7 @@
}
frameProcessed = true;
} catch (Exception e) {
+ e.printStackTrace();
if (feedPolicyAccessor.continueOnSoftFailure()) {
frame = exceptionHandler.handleException(e, frame);
if (frame == null) {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameCollection.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameCollection.java
similarity index 93%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameCollection.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameCollection.java
index 9ed547e..7980712 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameCollection.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameCollection.java
@@ -16,15 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import org.apache.asterix.common.feeds.api.IFeedMemoryComponent;
-import org.apache.asterix.common.feeds.api.IFeedMemoryManager;
+import org.apache.asterix.external.feed.api.IFeedMemoryComponent;
+import org.apache.asterix.external.feed.api.IFeedMemoryManager;
/**
* Represents an expandable collection of frames.
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameDistributor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
similarity index 97%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameDistributor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
index 9e106fb..543efb2 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameDistributor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -25,9 +25,10 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IFeedMemoryComponent.Type;
-import org.apache.asterix.common.feeds.api.IFeedMemoryManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedMemoryManager;
+import org.apache.asterix.external.feed.api.IFeedMemoryComponent.Type;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedId;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameEventCallback.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameEventCallback.java
similarity index 93%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameEventCallback.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameEventCallback.java
index 5551ce6..ba67862 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FrameEventCallback.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameEventCallback.java
@@ -16,13 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageFrameHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/StorageFrameHandler.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageFrameHandler.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/StorageFrameHandler.java
index 5b99b8c..22dcfac 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageFrameHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/StorageFrameHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.dataflow;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -24,7 +24,8 @@
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.asterix.common.feeds.FeedConstants.StatisticsConstants;
+import org.apache.asterix.external.feed.watch.IntakePartitionStatistics;
+import org.apache.asterix.external.util.FeedConstants.StatisticsConstants;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
public class StorageFrameHandler {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedCollectInfo.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedCollectInfo.java
similarity index 93%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedCollectInfo.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedCollectInfo.java
index aca6ab6..9f861d4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedCollectInfo.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedCollectInfo.java
@@ -16,14 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedId;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectionId.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionId.java
similarity index 82%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectionId.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionId.java
index 355d340..1af7153 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectionId.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionId.java
@@ -16,19 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.management;
import java.io.Serializable;
/**
- * A unique identifier for a feed connection. A feed connection is an instance of a data feed that is flowing into a dataset.
+ * A unique identifier for a feed connection. A feed connection is an instance of a data feed that is flowing into a
+ * dataset.
*/
public class FeedConnectionId implements Serializable {
private static final long serialVersionUID = 1L;
- private final FeedId feedId;
- private final String datasetName;
+ private final FeedId feedId; // Dataverse - Feed
+ private final String datasetName; // Dataset
public FeedConnectionId(FeedId feedId, String datasetName) {
this.feedId = feedId;
@@ -54,9 +55,8 @@
return false;
}
- if (this == o
- || (((FeedConnectionId) o).getFeedId().equals(feedId) && ((FeedConnectionId) o).getDatasetName()
- .equals(datasetName))) {
+ if (this == o || (((FeedConnectionId) o).getFeedId().equals(feedId)
+ && ((FeedConnectionId) o).getDatasetName().equals(datasetName))) {
return true;
}
return false;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedConnectionManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionManager.java
similarity index 91%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedConnectionManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionManager.java
index 7356c94..dd2fc60 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedConnectionManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.ArrayList;
import java.util.HashMap;
@@ -26,11 +26,9 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedRuntime;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeManager;
-import org.apache.asterix.common.feeds.api.IFeedConnectionManager;
+import org.apache.asterix.external.feed.api.IFeedConnectionManager;
+import org.apache.asterix.external.feed.runtime.FeedRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
/**
* An implementation of the IFeedManager interface.
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectionRequest.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectionRequest.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
index 6230eac..5f22a72 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectionRequest.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
@@ -16,15 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.List;
import java.util.Map;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
import org.apache.commons.lang3.StringUtils;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
-
/**
* A request for connecting a feed to a dataset.
*/
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedId.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedId.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedId.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedId.java
index 81b7c4e2..3145d72 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedId.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedId.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.management;
import java.io.Serializable;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedInfo.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedInfo.java
similarity index 96%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedInfo.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedInfo.java
index a247488..93f81d9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedInfo.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedInfo.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.external.feed.management;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobInfo;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedJointKey.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedJointKey.java
similarity index 97%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedJointKey.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedJointKey.java
index 8005967..7b08599 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedJointKey.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedJointKey.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.List;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedLifecycleEventSubscriber.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedLifecycleEventSubscriber.java
similarity index 94%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedLifecycleEventSubscriber.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedLifecycleEventSubscriber.java
index 2add90d..6e3cebce 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedLifecycleEventSubscriber.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedLifecycleEventSubscriber.java
@@ -16,13 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.Iterator;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
public class FeedLifecycleEventSubscriber implements IFeedLifecycleEventSubscriber {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedManager.java
similarity index 78%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedManager.java
index 7f9afb8..5095e7d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedManager.java
@@ -16,26 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.config.AsterixFeedProperties;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedMemoryManager;
-import org.apache.asterix.common.feeds.FeedMessageService;
-import org.apache.asterix.common.feeds.FeedMetricCollector;
-import org.apache.asterix.common.feeds.NodeLoadReportService;
-import org.apache.asterix.common.feeds.api.IFeedConnectionManager;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedMemoryManager;
-import org.apache.asterix.common.feeds.api.IFeedMessageService;
-import org.apache.asterix.common.feeds.api.IFeedMetadataManager;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
-import org.apache.asterix.common.feeds.api.IFeedSubscriptionManager;
-import org.apache.asterix.metadata.feeds.FeedConnectionManager;
-import org.apache.asterix.metadata.feeds.FeedSubscriptionManager;
+import org.apache.asterix.external.feed.api.IFeedConnectionManager;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedMemoryManager;
+import org.apache.asterix.external.feed.api.IFeedMessageService;
+import org.apache.asterix.external.feed.api.IFeedMetadataManager;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.api.IFeedSubscriptionManager;
+import org.apache.asterix.external.feed.message.FeedMessageService;
+import org.apache.asterix.external.feed.watch.FeedMetricCollector;
+import org.apache.asterix.external.feed.watch.NodeLoadReportService;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -68,14 +65,15 @@
private final int frameSize;
- public FeedManager(String nodeId, AsterixFeedProperties feedProperties, int frameSize) throws AsterixException, HyracksDataException {
+ public FeedManager(String nodeId, AsterixFeedProperties feedProperties, int frameSize)
+ throws AsterixException, HyracksDataException {
this.nodeId = nodeId;
this.feedSubscriptionManager = new FeedSubscriptionManager(nodeId);
this.feedConnectionManager = new FeedConnectionManager(nodeId);
this.feedMetadataManager = new FeedMetadataManager(nodeId);
this.feedMemoryManager = new FeedMemoryManager(nodeId, feedProperties, frameSize);
- String ccClusterIp = AsterixClusterProperties.INSTANCE.getCluster() != null ? AsterixClusterProperties.INSTANCE
- .getCluster().getMasterNode().getClusterIp() : "localhost";
+ String ccClusterIp = AsterixClusterProperties.INSTANCE.getCluster() != null
+ ? AsterixClusterProperties.INSTANCE.getCluster().getMasterNode().getClusterIp() : "localhost";
this.feedMessageService = new FeedMessageService(feedProperties, nodeId, ccClusterIp);
this.nodeLoadReportService = new NodeLoadReportService(nodeId, this);
try {
@@ -127,11 +125,6 @@
}
@Override
- public String getNodeId() {
- return nodeId;
- }
-
- @Override
public String toString() {
return "FeedManager " + "[" + nodeId + "]";
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMemoryManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedMemoryManager.java
similarity index 91%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMemoryManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedMemoryManager.java
index c39d82a..de9d22c 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMemoryManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedMemoryManager.java
@@ -16,16 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.config.AsterixFeedProperties;
-import org.apache.asterix.common.feeds.api.IFeedMemoryComponent;
-import org.apache.asterix.common.feeds.api.IFeedMemoryComponent.Type;
-import org.apache.asterix.common.feeds.api.IFeedMemoryManager;
+import org.apache.asterix.external.feed.api.IFeedMemoryComponent;
+import org.apache.asterix.external.feed.api.IFeedMemoryManager;
+import org.apache.asterix.external.feed.api.IFeedMemoryComponent.Type;
+import org.apache.asterix.external.feed.dataflow.DataBucketPool;
+import org.apache.asterix.external.feed.dataflow.FrameCollection;
public class FeedMemoryManager implements IFeedMemoryManager {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedMetadataManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedMetadataManager.java
similarity index 88%
rename from asterix-app/src/main/java/org/apache/asterix/feeds/FeedMetadataManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedMetadataManager.java
index 81cabeb..34ae461 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feeds/FeedMetadataManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedMetadataManager.java
@@ -16,18 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedMetadataManager;
-import org.apache.asterix.hyracks.bootstrap.FeedBootstrap;
-import org.apache.asterix.metadata.feeds.XAQLFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedMetadataManager;
+import org.apache.asterix.external.feed.message.XAQLFeedMessage;
+import org.apache.asterix.external.util.FeedConstants;
import org.apache.asterix.om.base.ARecord;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.base.IAObject;
@@ -50,7 +49,7 @@
IAType[] fieldTypes = new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING,
BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING };
- recordType = new ARecordType(FeedBootstrap.FAILED_TUPLE_DATASET_TYPE, fieldNames, fieldTypes, true);
+ recordType = new ARecordType(FeedConstants.FAILED_TUPLE_DATASET_TYPE, fieldNames, fieldTypes, true);
}
@Override
@@ -69,8 +68,8 @@
messageValue, dateTime };
ARecord record = new ARecord(recordType, fields);
StringBuilder builder = new StringBuilder();
- builder.append("use dataverse " + FeedBootstrap.FEEDS_METADATA_DV + ";" + "\n");
- builder.append("insert into dataset " + FeedBootstrap.FAILED_TUPLE_DATASET + " ");
+ builder.append("use dataverse " + FeedConstants.FEEDS_METADATA_DV + ";" + "\n");
+ builder.append("insert into dataset " + FeedConstants.FAILED_TUPLE_DATASET + " ");
builder.append(" (" + recordToString(record) + ")");
builder.append(";");
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedRuntimeManager.java
similarity index 91%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedRuntimeManager.java
index abd5daa..f7e98f7 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedRuntimeManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.management;
import java.io.IOException;
import java.util.Map;
@@ -27,7 +27,9 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IFeedConnectionManager;
+import org.apache.asterix.external.feed.api.IFeedConnectionManager;
+import org.apache.asterix.external.feed.runtime.FeedRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
public class FeedRuntimeManager {
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedSubscriptionManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedSubscriptionManager.java
similarity index 90%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedSubscriptionManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedSubscriptionManager.java
index b928e55..e402f92 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedSubscriptionManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedSubscriptionManager.java
@@ -16,16 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedSubscriptionManager;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.IFeedSubscriptionManager;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
public class FeedSubscriptionManager implements IFeedSubscriptionManager {
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedWorkManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedWorkManager.java
similarity index 86%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedWorkManager.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedWorkManager.java
index be12ff0..f2491db 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedWorkManager.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedWorkManager.java
@@ -16,14 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.management;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import org.apache.asterix.common.feeds.api.IFeedWork;
-import org.apache.asterix.common.feeds.api.IFeedWorkEventListener;
-import org.apache.asterix.common.feeds.api.IFeedWorkManager;
+import org.apache.asterix.external.feed.api.IFeedWork;
+import org.apache.asterix.external.feed.api.IFeedWorkEventListener;
+import org.apache.asterix.external.feed.api.IFeedWorkManager;
/**
* Handles asynchronous execution of feed management related tasks.
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/EndFeedMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/EndFeedMessage.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/EndFeedMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/EndFeedMessage.java
index 84db620..b0f7624 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/EndFeedMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/EndFeedMessage.java
@@ -16,15 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.util.FeedConstants;
/**
* A feed control message indicating the need to end the feed. This message is dispatched
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedCongestionMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedCongestionMessage.java
similarity index 88%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedCongestionMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedCongestionMessage.java
index c9e0fbd..6c924d2 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedCongestionMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedCongestionMessage.java
@@ -16,18 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedConstants.MessageConstants;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.util.FeedConstants;
+import org.apache.asterix.external.util.FeedConstants.MessageConstants;
public class FeedCongestionMessage extends FeedMessage {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedMessage.java
similarity index 92%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedMessage.java
index 11b1839..f2b354b 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedMessage.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
-import org.apache.asterix.common.feeds.api.IFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedMessage;
import org.apache.hyracks.api.dataflow.value.JSONSerializable;
/**
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMessageService.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedMessageService.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMessageService.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedMessageService.java
index 9582602..13d6622 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMessageService.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedMessageService.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.message;
import java.net.Socket;
import java.util.concurrent.ExecutorService;
@@ -29,8 +29,9 @@
import org.json.JSONObject;
import org.apache.asterix.common.config.AsterixFeedProperties;
-import org.apache.asterix.common.feeds.api.IFeedMessage;
-import org.apache.asterix.common.feeds.api.IFeedMessageService;
+import org.apache.asterix.external.feed.api.IFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedMessageService;
+import org.apache.asterix.external.util.FeedConstants;
/**
* Sends feed report messages on behalf of an operator instance
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedReportMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedReportMessage.java
similarity index 87%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedReportMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedReportMessage.java
index 1b14855..1b8c45d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/FeedReportMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedReportMessage.java
@@ -17,18 +17,17 @@
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedConstants.MessageConstants;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector.ValueType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector.ValueType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.util.FeedConstants;
+import org.apache.asterix.external.util.FeedConstants.MessageConstants;
public class FeedReportMessage extends FeedMessage {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedTupleCommitAckMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedTupleCommitAckMessage.java
similarity index 93%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedTupleCommitAckMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedTupleCommitAckMessage.java
index ada6566..61e26de 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedTupleCommitAckMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedTupleCommitAckMessage.java
@@ -16,14 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.message;
import javax.xml.bind.DatatypeConverter;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.message.FeedMessage;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.util.FeedConstants;
public class FeedTupleCommitAckMessage extends FeedMessage {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedTupleCommitResponseMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedTupleCommitResponseMessage.java
similarity index 92%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedTupleCommitResponseMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedTupleCommitResponseMessage.java
index cc32034..a61dc06 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedTupleCommitResponseMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/FeedTupleCommitResponseMessage.java
@@ -16,12 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.message.FeedMessage;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.util.FeedConstants;
public class FeedTupleCommitResponseMessage extends FeedMessage {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MessageListener.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageListener.java
similarity index 98%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/MessageListener.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageListener.java
index c2753d4..67f2884 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MessageListener.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageListener.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.message;
import java.io.IOException;
import java.io.InputStream;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MessageReceiver.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/MessageReceiver.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
index 6490c6a..abeb994 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MessageReceiver.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.message;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -24,7 +24,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IMessageReceiver;
+import org.apache.asterix.external.feed.api.IMessageReceiver;
public abstract class MessageReceiver<T> implements IMessageReceiver<T> {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/NodeReportMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/NodeReportMessage.java
similarity index 92%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/NodeReportMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/NodeReportMessage.java
index b45a966..1548d6d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/NodeReportMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/NodeReportMessage.java
@@ -16,13 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.api.IFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedMessage;
+import org.apache.asterix.external.util.FeedConstants;
public class NodeReportMessage extends FeedMessage {
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/PrepareStallMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/PrepareStallMessage.java
similarity index 91%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/PrepareStallMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/PrepareStallMessage.java
index 3ca1147..76fe0c2 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/PrepareStallMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/PrepareStallMessage.java
@@ -16,14 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.message.FeedMessage;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.util.FeedConstants;
/**
* A feed control message indicating the need to end the feed. This message is dispatched
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/RemoteSocketMessageListener.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/RemoteSocketMessageListener.java
similarity index 98%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/RemoteSocketMessageListener.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/RemoteSocketMessageListener.java
index 5c5c068..0749f82 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/RemoteSocketMessageListener.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/RemoteSocketMessageListener.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.message;
import java.io.IOException;
import java.io.InputStream;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/ScaleInReportMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/ScaleInReportMessage.java
similarity index 92%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/ScaleInReportMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/ScaleInReportMessage.java
index 6204704..62aba04 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/ScaleInReportMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/ScaleInReportMessage.java
@@ -16,15 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.util.FeedConstants;
/**
* A feed control message indicating the need to scale in a stage of the feed ingestion pipeline.
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/SocketMessageListener.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/SocketMessageListener.java
similarity index 97%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/SocketMessageListener.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/SocketMessageListener.java
index ef1d242..a1a0ad5 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/SocketMessageListener.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/SocketMessageListener.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.message;
import java.io.IOException;
import java.io.InputStream;
@@ -29,7 +29,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IMessageReceiver;
+import org.apache.asterix.external.feed.api.IMessageReceiver;
/**
* Listens for messages at a configured port and redirects them to a
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/StorageReportFeedMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/StorageReportFeedMessage.java
similarity index 94%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/StorageReportFeedMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/StorageReportFeedMessage.java
index 6f2c102..68ce74d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/StorageReportFeedMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/StorageReportFeedMessage.java
@@ -16,15 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedConstants.MessageConstants;
-import org.apache.asterix.common.feeds.FeedId;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.util.FeedConstants;
+import org.apache.asterix.external.util.FeedConstants.MessageConstants;
/**
* A feed control message sent from a storage runtime of a feed pipeline to report the intake timestamp corresponding
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/TerminateDataFlowMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/TerminateDataFlowMessage.java
similarity index 88%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/TerminateDataFlowMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/TerminateDataFlowMessage.java
index aaf20c5..ab77840 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/TerminateDataFlowMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/TerminateDataFlowMessage.java
@@ -17,14 +17,12 @@
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.message.FeedMessage;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.util.FeedConstants;
public class TerminateDataFlowMessage extends FeedMessage {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/ThrottlingEnabledFeedMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/ThrottlingEnabledFeedMessage.java
similarity index 89%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/message/ThrottlingEnabledFeedMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/ThrottlingEnabledFeedMessage.java
index 0ddd8ba..0459310 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/message/ThrottlingEnabledFeedMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/ThrottlingEnabledFeedMessage.java
@@ -16,16 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds.message;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.util.FeedConstants;
/**
* A feed control message indicating the need to end the feed. This message is dispatched
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/XAQLFeedMessage.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/XAQLFeedMessage.java
similarity index 90%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/XAQLFeedMessage.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/XAQLFeedMessage.java
index d6f7e0a..cef3fa9 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/XAQLFeedMessage.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/XAQLFeedMessage.java
@@ -16,14 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.feed.message;
import org.json.JSONException;
import org.json.JSONObject;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.message.FeedMessage;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.util.FeedConstants;
/**
* A feed control message indicating the need to execute a give AQL.
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedPolicy.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicy.java
similarity index 81%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedPolicy.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicy.java
index 4c2fe23..da5907c 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedPolicy.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicy.java
@@ -16,21 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.asterix.external.feed.policy;
-package org.apache.asterix.metadata.entities;
-
+import java.io.Serializable;
import java.util.Map;
-import org.apache.asterix.metadata.MetadataCache;
-import org.apache.asterix.metadata.api.IMetadataEntity;
-
-/**
- * Metadata describing a feed activity record.
- */
-public class FeedPolicy implements IMetadataEntity {
+public class FeedPolicy implements Serializable {
private static final long serialVersionUID = 1L;
-
private final String dataverseName;
// Enforced to be unique within a dataverse.
private final String policyName;
@@ -72,18 +65,6 @@
return true;
}
- @Override
- public Object addToCache(MetadataCache cache) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Object dropFromCache(MetadataCache cache) {
- // TODO Auto-generated method stub
- return null;
- }
-
public String getDescription() {
return description;
}
@@ -96,4 +77,4 @@
this.properties = properties;
}
-}
\ No newline at end of file
+}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedPolicyAccessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicyAccessor.java
similarity index 91%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedPolicyAccessor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicyAccessor.java
index cd7d598..077a58d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedPolicyAccessor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicyAccessor.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.policy;
import java.io.Serializable;
import java.util.Map;
@@ -28,7 +28,12 @@
private static final long serialVersionUID = 1L;
- /** failure configuration **/
+ /**
+ * --------------------------
+ * failure configuration
+ * --------------------------
+ **/
+
/** continue feed ingestion after a soft (runtime) failure **/
public static final String SOFT_FAILURE_CONTINUE = "soft.failure.continue";
@@ -44,7 +49,15 @@
/** framework provides guarantee that each received feed record will be processed through the ingestion pipeline at least once **/
public static final String AT_LEAST_ONE_SEMANTICS = "atleast.once.semantics";
- /** flow control configuration **/
+ /**
+ * --------------------------
+ * flow control configuration
+ * --------------------------
+ **/
+
+ /** enable buffering in feeds **/
+ public static final String BUFFERING_ENABLED = "buffering.enabled";
+
/** spill excess tuples to disk if an operator cannot process incoming data at its arrival rate **/
public static final String SPILL_TO_DISK_ON_CONGESTION = "spill.to.disk.on.congestion";
@@ -111,6 +124,10 @@
}
/** flow control **/
+ public boolean bufferingEnabled() {
+ return getBooleanPropertyValue(BUFFERING_ENABLED, false);
+ }
+
public boolean spillToDiskOnCongestion() {
return getBooleanPropertyValue(SPILL_TO_DISK_ON_CONGESTION, false);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feeds/FeedPolicyEnforcer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicyEnforcer.java
similarity index 90%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/feeds/FeedPolicyEnforcer.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicyEnforcer.java
index ae5c050..e0944ad 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feeds/FeedPolicyEnforcer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/policy/FeedPolicyEnforcer.java
@@ -16,14 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.feeds;
+package org.apache.asterix.external.feed.policy;
import java.rmi.RemoteException;
import java.util.Map;
import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
public class FeedPolicyEnforcer {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java
new file mode 100644
index 0000000..3ac28c9
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.feed.runtime;
+
+import org.apache.asterix.external.api.IAdapterRuntimeManager;
+import org.apache.asterix.external.api.IAdapterRuntimeManager.State;
+import org.apache.asterix.external.api.IFeedAdapter;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.util.ExternalDataExceptionUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * The class in charge of executing feed adapters.
+ */
+public class AdapterExecutor implements Runnable {
+
+ private static final Logger LOGGER = Logger.getLogger(AdapterExecutor.class.getName());
+
+ private final DistributeFeedFrameWriter writer; // A writer that sends frames to multiple receivers (that can
+ // increase or decrease at any time)
+ private final IFeedAdapter adapter; // The adapter
+ private final IAdapterRuntimeManager adapterManager;// The runtime manager <-- two way visibility -->
+
+ public AdapterExecutor(int partition, DistributeFeedFrameWriter writer, IFeedAdapter adapter,
+ IAdapterRuntimeManager adapterManager) {
+ this.writer = writer;
+ this.adapter = adapter;
+ this.adapterManager = adapterManager;
+ }
+
+ @Override
+ public void run() {
+ // Start by getting the partition number from the manager
+ int partition = adapterManager.getPartition();
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Starting ingestion for partition:" + partition);
+ }
+ boolean continueIngestion = true;
+ boolean failedIngestion = false;
+ while (continueIngestion) {
+ try {
+ // Start the adapter
+ adapter.start(partition, writer);
+ // Adapter has completed execution
+ continueIngestion = false;
+ } catch (Exception e) {
+ LOGGER.error("Exception during feed ingestion ", e);
+ // Check if the adapter wants to continue ingestion
+ if (ExternalDataExceptionUtils.isResolvable(e)) {
+ continueIngestion = adapter.handleException(e);
+ } else {
+ continueIngestion = false;
+ }
+ failedIngestion = !continueIngestion;
+ }
+ }
+ // Done with the adapter. about to close, setting the stage based on the failed ingestion flag and notifying the
+ // runtime manager
+ adapterManager.setState(failedIngestion ? State.FAILED_INGESTION : State.FINISHED_INGESTION);
+ synchronized (adapterManager) {
+ adapterManager.notifyAll();
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java
new file mode 100644
index 0000000..6c3e44d
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.feed.runtime;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.asterix.external.api.IAdapterRuntimeManager;
+import org.apache.asterix.external.api.IFeedAdapter;
+import org.apache.asterix.external.feed.api.IIntakeProgressTracker;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.log4j.Logger;
+
+/**
+ * This class manages the execution of an adapter within a feed
+ */
+public class AdapterRuntimeManager implements IAdapterRuntimeManager {
+
+ private static final Logger LOGGER = Logger.getLogger(AdapterRuntimeManager.class.getName());
+
+ private final FeedId feedId; // (dataverse-feed)
+
+ private final IFeedAdapter feedAdapter; // The adapter
+
+ private final IIntakeProgressTracker tracker; // Not used. needs to be fixed soon.
+
+ private final AdapterExecutor adapterExecutor; // The executor for the adapter <-- two way visibility -->
+
+ private final int partition; // The partition number
+
+ private final ExecutorService executorService; // Executor service to run/shutdown the adapter executor
+
+ private IngestionRuntime ingestionRuntime; // Runtime representing the ingestion stage of a feed <-- two way
+ // visibility -->
+
+ private State state; // One of {ACTIVE_INGESTION, NACTIVE_INGESTION, FINISHED_INGESTION,
+ // FAILED_INGESTION}
+
+ public AdapterRuntimeManager(FeedId feedId, IFeedAdapter feedAdapter, IIntakeProgressTracker tracker,
+ DistributeFeedFrameWriter writer, int partition) {
+ this.feedId = feedId;
+ this.feedAdapter = feedAdapter;
+ this.tracker = tracker;
+ this.partition = partition;
+ this.adapterExecutor = new AdapterExecutor(partition, writer, feedAdapter, this);
+ this.executorService = Executors.newSingleThreadExecutor();
+ this.state = State.INACTIVE_INGESTION;
+ }
+
+ @Override
+ public void start() throws Exception {
+ state = State.ACTIVE_INGESTION;
+ executorService.execute(adapterExecutor);
+ }
+
+ @Override
+ public void stop() {
+ boolean stopped = false;
+ try {
+ stopped = feedAdapter.stop();
+ } catch (Exception exception) {
+ LOGGER.error("Unable to stop adapter " + feedAdapter, exception);
+ } finally {
+ state = State.FINISHED_INGESTION;
+ if (stopped) {
+ // stop() returned true, we wait for the process termination
+ executorService.shutdown();
+ try {
+ executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ LOGGER.error("Interrupted while waiting for feed adapter to finish its work", e);
+ }
+ } else {
+ // stop() returned false, we try to force shutdown
+ executorService.shutdownNow();
+ }
+
+ }
+ }
+
+ @Override
+ public FeedId getFeedId() {
+ return feedId;
+ }
+
+ @Override
+ public String toString() {
+ return feedId + "[" + partition + "]";
+ }
+
+ @Override
+ public IFeedAdapter getFeedAdapter() {
+ return feedAdapter;
+ }
+
+ public IIntakeProgressTracker getTracker() {
+ return tracker;
+ }
+
+ @Override
+ public synchronized State getState() {
+ return state;
+ }
+
+ @Override
+ public synchronized void setState(State state) {
+ this.state = state;
+ }
+
+ public AdapterExecutor getAdapterExecutor() {
+ return adapterExecutor;
+ }
+
+ @Override
+ public int getPartition() {
+ return partition;
+ }
+
+ public IngestionRuntime getIngestionRuntime() {
+ return ingestionRuntime;
+ }
+
+ @Override
+ public IIntakeProgressTracker getProgressTracker() {
+ return tracker;
+ }
+
+}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/CollectionRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java
similarity index 76%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/CollectionRuntime.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java
index 9865501..967dc3e 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/CollectionRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java
@@ -16,13 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.runtime;
import java.util.Map;
-import org.apache.asterix.common.feeds.FeedFrameCollector.State;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
-import org.apache.asterix.common.feeds.api.ISubscriberRuntime;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.ISubscriberRuntime;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector.State;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.hyracks.api.comm.IFrameWriter;
/**
@@ -33,10 +36,10 @@
*/
public class CollectionRuntime extends FeedRuntime implements ISubscriberRuntime {
- private final FeedConnectionId connectionId;
- private final ISubscribableRuntime sourceRuntime;
- private final Map<String, String> feedPolicy;
- private FeedFrameCollector frameCollector;
+ private final FeedConnectionId connectionId; // [Dataverse - Feed - Dataset]
+ private final ISubscribableRuntime sourceRuntime; // Runtime that provides the data
+ private final Map<String, String> feedPolicy; // Policy associated with the feed
+ private FeedFrameCollector frameCollector; // Collector that can be plugged into a frame distributor
public CollectionRuntime(FeedConnectionId connectionId, FeedRuntimeId runtimeId,
FeedRuntimeInputHandler inputSideHandler, IFrameWriter outputSideWriter, ISubscribableRuntime sourceRuntime,
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/FeedRuntime.java
similarity index 88%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntime.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/FeedRuntime.java
index 87276ec..76b1b19 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/FeedRuntime.java
@@ -16,10 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.runtime;
-import org.apache.asterix.common.feeds.api.IFeedOperatorOutputSideHandler;
-import org.apache.asterix.common.feeds.api.IFeedRuntime;
+import org.apache.asterix.external.feed.api.IFeedOperatorOutputSideHandler;
+import org.apache.asterix.external.feed.api.IFeedRuntime;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
import org.apache.hyracks.api.comm.IFrameWriter;
public class FeedRuntime implements IFeedRuntime {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeId.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/FeedRuntimeId.java
similarity index 94%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeId.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/FeedRuntimeId.java
index bf3c2c1..45d8afe 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedRuntimeId.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/FeedRuntimeId.java
@@ -16,11 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.runtime;
import java.io.Serializable;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
public class FeedRuntimeId implements Serializable {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IngestionRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
similarity index 82%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/IngestionRuntime.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
index 926df39..fd6fcb3 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IngestionRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
@@ -16,11 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.runtime;
import java.util.logging.Level;
-import org.apache.asterix.common.feeds.api.IAdapterRuntimeManager;
+import org.apache.asterix.external.api.IAdapterRuntimeManager;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
+import org.apache.asterix.external.feed.dataflow.FrameDistributor;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
public class IngestionRuntime extends SubscribableRuntime {
@@ -33,11 +38,12 @@
this.adapterRuntimeManager = adaptorRuntimeManager;
}
+ @Override
public void subscribeFeed(FeedPolicyAccessor fpa, CollectionRuntime collectionRuntime) throws Exception {
FeedFrameCollector reader = dWriter.subscribeFeed(fpa, collectionRuntime.getInputHandler(),
collectionRuntime.getConnectionId());
collectionRuntime.setFrameCollector(reader);
-
+
if (dWriter.getDistributionMode().equals(FrameDistributor.DistributionMode.SINGLE)) {
adapterRuntimeManager.start();
}
@@ -47,16 +53,18 @@
}
}
+ @Override
public void unsubscribeFeed(CollectionRuntime collectionRuntime) throws Exception {
- dWriter.unsubscribeFeed(collectionRuntime.getInputHandler());
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Unsubscribed feed collection [" + collectionRuntime + "] from " + this);
- }
- if (dWriter.getDistributionMode().equals(FrameDistributor.DistributionMode.INACTIVE)) {
+ if (dWriter.getDistributionMode().equals(FrameDistributor.DistributionMode.SINGLE)) {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Stopping adapter for " + this + " as no more registered collectors");
}
adapterRuntimeManager.stop();
+ } else {
+ dWriter.unsubscribeFeed(collectionRuntime.getInputHandler());
+ }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Unsubscribed feed collection [" + collectionRuntime + "] from " + this);
}
subscribers.remove(collectionRuntime);
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SubscribableFeedRuntimeId.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableFeedRuntimeId.java
similarity index 89%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/SubscribableFeedRuntimeId.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableFeedRuntimeId.java
index 7eb5921..f6db99c 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SubscribableFeedRuntimeId.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableFeedRuntimeId.java
@@ -16,9 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.runtime;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedId;
public class SubscribableFeedRuntimeId extends FeedRuntimeId {
private static final long serialVersionUID = 1L;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SubscribableRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java
similarity index 85%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/SubscribableRuntime.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java
index b09b3ff..056875c 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SubscribableRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java
@@ -16,14 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.runtime;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
-import org.apache.asterix.common.feeds.api.ISubscriberRuntime;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.ISubscriberRuntime;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
public class SubscribableRuntime extends FeedRuntime implements ISubscribableRuntime {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/BasicMonitoredBuffer.java
similarity index 78%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/BasicMonitoredBuffer.java
index 70833fc..ad40608 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/BasicMonitoredBuffer.java
@@ -16,13 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
-import org.apache.asterix.common.feeds.api.IExceptionHandler;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback;
-import org.apache.asterix.common.feeds.api.IFramePostProcessor;
-import org.apache.asterix.common.feeds.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.api.IExceptionHandler;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/ComputeSideMonitoredBuffer.java
similarity index 78%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/ComputeSideMonitoredBuffer.java
index 7ec3fdf..211fc7b 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/ComputeSideMonitoredBuffer.java
@@ -16,13 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
-import org.apache.asterix.common.feeds.api.IExceptionHandler;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback;
-import org.apache.asterix.common.feeds.api.IFramePostProcessor;
-import org.apache.asterix.common.feeds.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.api.IExceptionHandler;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedActivity.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedActivity.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedActivity.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedActivity.java
index 11130a1..fc06ab9 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedActivity.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedActivity.java
@@ -1,23 +1,21 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
+ * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
+ * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
+ * KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.Map;
@@ -37,7 +35,6 @@
public static final String COLLECT_LOCATIONS = "collect-locations";
public static final String FEED_POLICY_NAME = "feed-policy-name";
public static final String FEED_CONNECT_TIMESTAMP = "feed-connect-timestamp";
-
}
public FeedActivity(String dataverseName, String feedName, String datasetName,
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectJobInfo.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedConnectJobInfo.java
similarity index 93%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectJobInfo.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedConnectJobInfo.java
index 4d6a427..3e42169 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConnectJobInfo.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedConnectJobInfo.java
@@ -16,12 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedIntakeInfo.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedIntakeInfo.java
similarity index 91%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedIntakeInfo.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedIntakeInfo.java
index 0382b23..3b11811 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedIntakeInfo.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedIntakeInfo.java
@@ -16,11 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.List;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.management.FeedId;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedJobInfo.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedJobInfo.java
similarity index 97%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedJobInfo.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedJobInfo.java
index 2db9955..92e00cb 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedJobInfo.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedJobInfo.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.logging.Level;
import java.util.logging.Logger;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMetricCollector.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedMetricCollector.java
similarity index 95%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMetricCollector.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedMetricCollector.java
index 7b76692..f0db639 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedMetricCollector.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/FeedMetricCollector.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.HashMap;
import java.util.Map;
@@ -24,7 +24,9 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
public class FeedMetricCollector implements IFeedMetricCollector {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakePartitionStatistics.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/IntakePartitionStatistics.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakePartitionStatistics.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/IntakePartitionStatistics.java
index 5601f73..acfd1fb 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakePartitionStatistics.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/IntakePartitionStatistics.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.BitSet;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/IntakeSideMonitoredBuffer.java
similarity index 78%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/IntakeSideMonitoredBuffer.java
index 10b7ddb..7a79e23 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/IntakeSideMonitoredBuffer.java
@@ -16,13 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
-import org.apache.asterix.common.feeds.api.IExceptionHandler;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback;
-import org.apache.asterix.common.feeds.api.IFramePostProcessor;
-import org.apache.asterix.common.feeds.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.api.IExceptionHandler;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
index e5a22b5..db38edf 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -24,19 +24,27 @@
import java.util.TimerTask;
import java.util.logging.Level;
-import org.apache.asterix.common.feeds.MonitoredBufferTimerTasks.LogInputOutputRateTask;
-import org.apache.asterix.common.feeds.MonitoredBufferTimerTasks.MonitorInputQueueLengthTimerTask;
-import org.apache.asterix.common.feeds.MonitoredBufferTimerTasks.MonitoreProcessRateTimerTask;
-import org.apache.asterix.common.feeds.MonitoredBufferTimerTasks.MonitoredBufferStorageTimerTask;
-import org.apache.asterix.common.feeds.api.IExceptionHandler;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector.MetricType;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector.ValueType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback.FrameEvent;
-import org.apache.asterix.common.feeds.api.IFramePostProcessor;
-import org.apache.asterix.common.feeds.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.api.IExceptionHandler;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector.MetricType;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector.ValueType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IFrameEventCallback.FrameEvent;
+import org.apache.asterix.external.feed.dataflow.DataBucket;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.dataflow.StorageFrameHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.MessageReceiver;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.feed.watch.MonitoredBufferTimerTasks.LogInputOutputRateTask;
+import org.apache.asterix.external.feed.watch.MonitoredBufferTimerTasks.MonitorInputQueueLengthTimerTask;
+import org.apache.asterix.external.feed.watch.MonitoredBufferTimerTasks.MonitoreProcessRateTimerTask;
+import org.apache.asterix.external.feed.watch.MonitoredBufferTimerTasks.MonitoredBufferStorageTimerTask;
+import org.apache.asterix.external.util.FeedFrameUtil;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
@@ -212,7 +220,7 @@
}
protected void preProcessFrame(ByteBuffer frame) throws Exception {
- if (postProcessor == null) {
+ if (preProcessor == null) {
preProcessor = getFramePreProcessor();
}
if (preProcessor != null) {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBufferTimerTasks.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
similarity index 91%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBufferTimerTasks.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
index 3434b4f..86c6bca 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBufferTimerTasks.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.ArrayList;
import java.util.HashMap;
@@ -29,16 +29,21 @@
import java.util.logging.Logger;
import org.apache.asterix.common.config.AsterixFeedProperties;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedMessageService;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector.ValueType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback.FrameEvent;
-import org.apache.asterix.common.feeds.message.FeedReportMessage;
-import org.apache.asterix.common.feeds.message.ScaleInReportMessage;
-import org.apache.asterix.common.feeds.message.StorageReportFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedMessageService;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector.ValueType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IFrameEventCallback.FrameEvent;
+import org.apache.asterix.external.feed.dataflow.StorageFrameHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.FeedReportMessage;
+import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
+import org.apache.asterix.external.feed.message.FeedTupleCommitResponseMessage;
+import org.apache.asterix.external.feed.message.ScaleInReportMessage;
+import org.apache.asterix.external.feed.message.StorageReportFeedMessage;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
public class MonitoredBufferTimerTasks {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoad.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoad.java
similarity index 92%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoad.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoad.java
index b654563..d3919b5 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoad.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoad.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
public class NodeLoad implements Comparable<NodeLoad> {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoadReport.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoadReport.java
similarity index 96%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoadReport.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoadReport.java
index a509341..bfddcf6 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoadReport.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoadReport.java
@@ -16,8 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
+import org.apache.asterix.external.util.FeedConstants;
import org.json.JSONException;
import org.json.JSONObject;
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoadReportService.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoadReportService.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoadReportService.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoadReportService.java
index 6be0211..f651935 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/NodeLoadReportService.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/NodeLoadReportService.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
@@ -25,10 +25,11 @@
import java.util.Timer;
import java.util.TimerTask;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedMessageService;
-import org.apache.asterix.common.feeds.api.IFeedService;
-import org.apache.asterix.common.feeds.message.NodeReportMessage;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedMessageService;
+import org.apache.asterix.external.feed.api.IFeedService;
+import org.apache.asterix.external.feed.message.NodeReportMessage;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
public class NodeLoadReportService implements IFeedService {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/Series.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/Series.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/Series.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/Series.java
index 6f438ad..ec95371 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/Series.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/Series.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector.MetricType;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector.MetricType;
public abstract class Series {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SeriesAvg.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/SeriesAvg.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/SeriesAvg.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/SeriesAvg.java
index 6bfe925..f75379d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SeriesAvg.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/SeriesAvg.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector.MetricType;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector.MetricType;
public class SeriesAvg extends Series {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SeriesRate.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/SeriesRate.java
similarity index 94%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/SeriesRate.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/SeriesRate.java
index 9d0b0ea..91eea87 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/SeriesRate.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/SeriesRate.java
@@ -16,12 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.util.Timer;
import java.util.TimerTask;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector.MetricType;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector.MetricType;
public class SeriesRate extends Series {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/StorageSideMonitoredBuffer.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/StorageSideMonitoredBuffer.java
index 4027237..1f9551d 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/StorageSideMonitoredBuffer.java
@@ -16,16 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.feed.watch;
import java.nio.ByteBuffer;
-import org.apache.asterix.common.feeds.FeedConstants.StatisticsConstants;
-import org.apache.asterix.common.feeds.api.IExceptionHandler;
-import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
-import org.apache.asterix.common.feeds.api.IFrameEventCallback;
-import org.apache.asterix.common.feeds.api.IFramePostProcessor;
-import org.apache.asterix.common.feeds.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.api.IExceptionHandler;
+import org.apache.asterix.external.feed.api.IFeedMetricCollector;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.dataflow.StorageFrameHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.util.FeedConstants.StatisticsConstants;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/AbstractStreamRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/AbstractStreamRecordReader.java
index 3b59b98..93ba0a0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/AbstractStreamRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/AbstractStreamRecordReader.java
@@ -37,6 +37,7 @@
protected int bufferLength = 0;
protected int bufferPosn = 0;
protected IExternalIndexer indexer;
+ protected boolean done = false;
@Override
public IRawRecord<char[]> next() throws IOException {
@@ -45,7 +46,10 @@
@Override
public void close() throws IOException {
- reader.close();
+ if (!done) {
+ reader.close();
+ }
+ done = true;
}
public void setInputStream(AInputStream inputStream) throws IOException {
@@ -72,4 +76,15 @@
public void setIndexer(IExternalIndexer indexer) {
this.indexer = indexer;
}
+
+ @Override
+ public boolean stop() {
+ try {
+ reader.stop();
+ return true;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/LineRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/LineRecordReader.java
index 9b11df6..2b33d7a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/LineRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/LineRecordReader.java
@@ -32,6 +32,9 @@
@Override
public boolean hasNext() throws IOException {
+ if (done) {
+ return false;
+ }
/* We're reading data from in, but the head of the stream may be
* already buffered in buffer, so we have several cases:
* 1. No newline characters are in the buffer, so we need to copy
@@ -63,7 +66,7 @@
recordNumber++;
return true;
}
- reader.close();
+ close();
return false; //EOF
}
}
@@ -92,11 +95,6 @@
}
@Override
- public boolean stop() {
- return false;
- }
-
- @Override
public void configure(Map<String, String> configuration) throws Exception {
super.configure(configuration);
if (ExternalDataUtils.hasHeader(configuration)) {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/QuotedLineRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/QuotedLineRecordReader.java
index 668876e..49e67e9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/QuotedLineRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/QuotedLineRecordReader.java
@@ -44,6 +44,9 @@
@Override
public boolean hasNext() throws IOException {
+ if (done) {
+ return false;
+ }
newlineLength = 0;
prevCharCR = false;
prevCharEscape = false;
@@ -65,6 +68,7 @@
recordNumber++;
return true;
}
+ close();
return false;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/SemiStructuredRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/SemiStructuredRecordReader.java
index 9864805..84c96d0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/SemiStructuredRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/SemiStructuredRecordReader.java
@@ -67,6 +67,9 @@
@Override
public boolean hasNext() throws Exception {
+ if (done) {
+ return false;
+ }
record.reset();
boolean hasStarted = false;
boolean hasFinished = false;
@@ -79,6 +82,7 @@
startPosn = bufferPosn = 0;
bufferLength = reader.read(inputBuffer);
if (bufferLength <= 0) {
+ close();
return false; // EOF
}
}
@@ -142,6 +146,12 @@
@Override
public boolean stop() {
- return false;
+ try {
+ reader.stop();
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ return true;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TwitterPushRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TwitterPushRecordReader.java
index e7c141d..3ce6a81 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TwitterPushRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TwitterPushRecordReader.java
@@ -38,11 +38,16 @@
private LinkedBlockingQueue<Status> inputQ;
private TwitterStream twitterStream;
private GenericRecord<Status> record;
+ private boolean closed = false;
@Override
public void close() throws IOException {
- twitterStream.clearListeners();
- twitterStream.cleanUp();
+ if (!closed) {
+ twitterStream.clearListeners();
+ twitterStream.cleanUp();
+ twitterStream = null;
+ closed = true;
+ }
}
@Override
@@ -61,7 +66,7 @@
@Override
public boolean hasNext() throws Exception {
- return true;
+ return !closed;
}
@Override
@@ -81,7 +86,12 @@
@Override
public boolean stop() {
- return false;
+ try {
+ close();
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
}
private class TweetListener implements StatusListener {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/factory/TwitterRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/factory/TwitterRecordReaderFactory.java
index 72aaa37..6840c11 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/factory/TwitterRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/factory/TwitterRecordReaderFactory.java
@@ -97,7 +97,7 @@
pull = false;
} else {
throw new AsterixException("One of boolean parameters " + ExternalDataConstants.KEY_PULL + " and "
- + ExternalDataConstants.KEY_PUSH + "must be specified as part of adaptor configuration");
+ + ExternalDataConstants.KEY_PUSH + " must be specified as part of adaptor configuration");
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
index e573f74..7ba6032 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.input.stream;
+import java.io.IOException;
import java.io.InputStreamReader;
public class AInputStreamReader extends InputStreamReader {
@@ -31,4 +32,12 @@
public boolean skipError() throws Exception {
return in.skipError();
}
+
+ public void stop() throws IOException {
+ try {
+ in.stop();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStreamProvider.java
index b3ad1c3..8f4c094 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStreamProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStreamProvider.java
@@ -63,7 +63,7 @@
}
} else if (value.getLength() == pos) {
pos++;
- return ExternalDataConstants.EOL;
+ return ExternalDataConstants.BYTE_LF;
}
return value.getBytes()[pos++];
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStreamProvider.java
index b511617..22d0a87 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStreamProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStreamProvider.java
@@ -19,38 +19,44 @@
package org.apache.asterix.external.input.stream;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
+import java.nio.file.Path;
import java.util.Map;
import org.apache.asterix.external.api.IInputStreamProvider;
+import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.util.FeedUtils;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.dataflow.std.file.FileSplit;
public class LocalFSInputStreamProvider implements IInputStreamProvider {
- private FileSplit[] fileSplits;
- private int partition;
+ private String expression;
+ private boolean isFeed;
+ private Path path;
+ private File feedLogFile;
public LocalFSInputStreamProvider(FileSplit[] fileSplits, IHyracksTaskContext ctx,
- Map<String, String> configuration, int partition) {
- this.partition = partition;
- this.fileSplits = fileSplits;
- }
+ Map<String, String> configuration, int partition, String expression, boolean isFeed,
+ FileSplit[] feedLogFileSplits) {
+ this.expression = expression;
+ this.isFeed = isFeed;
+ this.path = fileSplits[partition].getLocalFile().getFile().toPath();
+ if (feedLogFileSplits != null) {
+ this.feedLogFile = FeedUtils
+ .getAbsoluteFileRef(feedLogFileSplits[partition].getLocalFile().getFile().getPath(),
+ feedLogFileSplits[partition].getIODeviceId(), ctx.getIOManager())
+ .getFile();
- @Override
- public AInputStream getInputStream() throws Exception {
- FileSplit split = fileSplits[partition];
- File inputFile = split.getLocalFile().getFile();
- InputStream in;
- try {
- in = new FileInputStream(inputFile);
- return new BasicInputStream(in);
- } catch (FileNotFoundException e) {
- throw new IOException(e);
}
}
+ @Override
+ public AInputStream getInputStream() throws IOException {
+ FeedLogManager feedLogManager = null;
+ if (isFeed && feedLogFile != null) {
+ feedLogManager = new FeedLogManager(feedLogFile);
+ }
+ return new LocalFileSystemInputStream(path, expression, feedLogManager, isFeed);
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
new file mode 100644
index 0000000..7eebe4c
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.util.FileSystemWatcher;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class LocalFileSystemInputStream extends AInputStream {
+ private final FileSystemWatcher watcher;
+ private FileInputStream in;
+ private byte lastByte;
+
+ public LocalFileSystemInputStream(Path inputResource, String expression, FeedLogManager logManager, boolean isFeed)
+ throws IOException {
+ this.watcher = new FileSystemWatcher(logManager, inputResource, expression, isFeed);
+ this.watcher.init();
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOException ioe = null;
+ if (in != null) {
+ try {
+ closeFile();
+ } catch (Exception e) {
+ ioe = new IOException(e);
+ }
+ }
+ try {
+ watcher.close();
+ } catch (Exception e) {
+ if (ioe == null) {
+ throw e;
+ }
+ ioe.addSuppressed(e);
+ throw ioe;
+ }
+ }
+
+ private void closeFile() throws IOException {
+ if (in != null) {
+ try {
+ in.close();
+ } finally {
+ in = null;
+ }
+ }
+ }
+
+ /**
+ * Closes the current input stream and opens the next one, if any.
+ */
+ private boolean advance() throws IOException {
+ closeFile();
+ if (watcher.hasNext()) {
+ in = new FileInputStream(watcher.next());
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int read() throws IOException {
+ throw new HyracksDataException(
+ "read() is not supported with this stream. use read(byte[] b, int off, int len)");
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (in == null) {
+ if (!advance()) {
+ return -1;
+ }
+ }
+ int result = in.read(b, off, len);
+ while (result < 0 && advance()) {
+ // return a new line at the end of every file <--Might create problems for some cases depending on the parser implementation-->
+ if (lastByte != ExternalDataConstants.BYTE_LF && lastByte != ExternalDataConstants.BYTE_LF) {
+ lastByte = ExternalDataConstants.BYTE_LF;
+ b[off] = ExternalDataConstants.BYTE_LF;
+ return 1;
+ }
+ // recursive call
+ result = in.read(b, off, len);
+ }
+ if (result > 0) {
+ lastByte = b[off + result - 1];
+ }
+ return result;
+ }
+
+ @Override
+ public boolean skipError() throws Exception {
+ advance();
+ return true;
+ }
+
+ @Override
+ public boolean stop() throws Exception {
+ watcher.close();
+ return true;
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStreamProvider.java
index d32a94f..7c64aa3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStreamProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStreamProvider.java
@@ -30,7 +30,7 @@
import java.util.logging.Logger;
import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.runtime.TweetGenerator;
+import org.apache.asterix.external.util.TweetGenerator;
import org.apache.hyracks.api.context.IHyracksTaskContext;
public class TwitterFirehoseInputStreamProvider implements IInputStreamProvider {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
index 14c712a..ab1f8a0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
@@ -29,8 +29,10 @@
import org.apache.asterix.external.api.INodeResolver;
import org.apache.asterix.external.api.INodeResolverFactory;
import org.apache.asterix.external.input.stream.LocalFSInputStreamProvider;
-import org.apache.asterix.external.util.DNSResolverFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.FeedUtils;
+import org.apache.asterix.external.util.NodeResolverFactory;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -41,15 +43,21 @@
private static final long serialVersionUID = 1L;
- protected static final INodeResolver DEFAULT_NODE_RESOLVER = new DNSResolverFactory().createNodeResolver();
+ protected static final INodeResolver DEFAULT_NODE_RESOLVER = new NodeResolverFactory().createNodeResolver();
protected static final Logger LOGGER = Logger.getLogger(LocalFSInputStreamProviderFactory.class.getName());
protected static INodeResolver nodeResolver;
protected Map<String, String> configuration;
- protected FileSplit[] fileSplits;
+ protected FileSplit[] inputFileSplits;
+ protected FileSplit[] feedLogFileSplits; // paths where instances of this feed can use as log storage
+ protected boolean isFeed;
+ protected String expression;
+ // transient fields (They don't need to be serialized and transferred)
+ private transient AlgebricksAbsolutePartitionConstraint constraints;
@Override
public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition) throws Exception {
- return new LocalFSInputStreamProvider(fileSplits, ctx, configuration, partition);
+ return new LocalFSInputStreamProvider(inputFileSplits, ctx, configuration, partition, expression, isFeed,
+ feedLogFileSplits);
}
@Override
@@ -67,16 +75,23 @@
this.configuration = configuration;
String[] splits = configuration.get(ExternalDataConstants.KEY_PATH).split(",");
configureFileSplits(splits);
+ configurePartitionConstraint();
+ this.isFeed = ExternalDataUtils.isFeed(configuration) && ExternalDataUtils.keepDataSourceOpen(configuration);
+ if (isFeed) {
+ feedLogFileSplits = FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
+ ExternalDataUtils.getFeedName(configuration), constraints);
+ }
+ this.expression = configuration.get(ExternalDataConstants.KEY_EXPRESSION);
}
@Override
public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
- return configurePartitionConstraint();
+ return constraints;
}
private void configureFileSplits(String[] splits) throws AsterixException {
- if (fileSplits == null) {
- fileSplits = new FileSplit[splits.length];
+ if (inputFileSplits == null) {
+ inputFileSplits = new FileSplit[splits.length];
String nodeName;
String nodeLocalPath;
int count = 0;
@@ -90,19 +105,19 @@
nodeName = trimmedValue.split(":")[0];
nodeLocalPath = trimmedValue.split("://")[1];
FileSplit fileSplit = new FileSplit(nodeName, new FileReference(new File(nodeLocalPath)));
- fileSplits[count++] = fileSplit;
+ inputFileSplits[count++] = fileSplit;
}
}
}
- private AlgebricksPartitionConstraint configurePartitionConstraint() throws AsterixException {
- String[] locs = new String[fileSplits.length];
+ private void configurePartitionConstraint() throws AsterixException {
+ String[] locs = new String[inputFileSplits.length];
String location;
- for (int i = 0; i < fileSplits.length; i++) {
- location = getNodeResolver().resolveNode(fileSplits[i].getNodeName());
+ for (int i = 0; i < inputFileSplits.length; i++) {
+ location = getNodeResolver().resolveNode(inputFileSplits[i].getNodeName());
locs[i] = location;
}
- return new AlgebricksAbsolutePartitionConstraint(locs);
+ constraints = new AlgebricksAbsolutePartitionConstraint(locs);
}
protected INodeResolver getNodeResolver() {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java b/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
index e9c15cb..fd3d9e3 100755
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
@@ -75,7 +75,7 @@
}
}
- public static ISerializerDeserializer getSerDe(Object typeInfo) {
+ public static ISerializerDeserializer<?> getSerDe(Object typeInfo) {
return AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(typeInfo);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/library/ResultCollector.java b/asterix-external-data/src/main/java/org/apache/asterix/external/library/ResultCollector.java
index 192bd4e..b6795f6 100755
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/library/ResultCollector.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/library/ResultCollector.java
@@ -117,6 +117,7 @@
return reusableResultObjectHolder;
}
+ @SuppressWarnings("unchecked")
private void serializeResult(IAObject object) throws AsterixException {
try {
AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(finfo.getReturnType())
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java b/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java
index 93b4bf1..e7c1ec1 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java
@@ -23,7 +23,6 @@
import java.util.List;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.SerializerDeserializerUtil;
import org.apache.asterix.external.api.IJObject;
@@ -65,7 +64,6 @@
/**
* Normalize an input string by removing linebreaks, and replace them with space
* Also remove non-readable special characters
- *
* @param originalString
* The input String
* @return
@@ -314,54 +312,38 @@
int numberOfSchemaFields = recordType.getFieldTypes().length;
byte[] recordBits = dis.getInputStream().getArray();
boolean isExpanded = false;
- int s = dis.getInputStream().getPosition();
- int recordOffset = s;
- int openPartOffset = 0;
- int offsetArrayOffset = 0;
+ dis.getInputStream();
int[] fieldOffsets = new int[numberOfSchemaFields];
IJObject[] closedFields = new IJObject[numberOfSchemaFields];
- if (recordType == null) {
- openPartOffset = s + AInt32SerializerDeserializer.getInt(recordBits, s + 6);
- s += 8;
- isExpanded = true;
- } else {
- dis.skip(4); // reading length is not required.
- if (recordType.isOpen()) {
- isExpanded = dis.readBoolean();
- if (isExpanded) {
- openPartOffset = s + dis.readInt(); // AInt32SerializerDeserializer.getInt(recordBits, s + 6);
- } else {
- // do nothing s += 6;
- }
+ dis.skip(4); // reading length is not required.
+ if (recordType.isOpen()) {
+ isExpanded = dis.readBoolean();
+ if (isExpanded) {
+ dis.readInt();
} else {
- // do nothing s += 5;
}
+ } else {
}
if (numberOfSchemaFields > 0) {
- int numOfSchemaFields = dis.readInt(); //s += 4;
+ dis.readInt();
int nullBitMapOffset = 0;
boolean hasNullableFields = NonTaggedFormatUtil.hasNullableField(recordType);
if (hasNullableFields) {
- nullBitMapOffset = dis.getInputStream().getPosition();//s
- offsetArrayOffset = dis.getInputStream().getPosition() //s
- + (numberOfSchemaFields % 8 == 0 ? numberOfSchemaFields / 8
- : numberOfSchemaFields / 8 + 1);
+ nullBitMapOffset = dis.getInputStream().getPosition();
+ dis.getInputStream();
} else {
- offsetArrayOffset = dis.getInputStream().getPosition();
+ dis.getInputStream();
}
for (int i = 0; i < numberOfSchemaFields; i++) {
- fieldOffsets[i] = dis.readInt(); // AInt32SerializerDeserializer.getInt(recordBits, offsetArrayOffset) + recordOffset;
- // offsetArrayOffset += 4;
+ fieldOffsets[i] = dis.readInt();
}
for (int fieldNumber = 0; fieldNumber < numberOfSchemaFields; fieldNumber++) {
if (hasNullableFields) {
byte b1 = recordBits[nullBitMapOffset + fieldNumber / 8];
int p = 1 << (7 - (fieldNumber % 8));
if ((b1 & p) == 0) {
- // set null value (including type tag inside)
- //fieldValues.add(nullReference);
continue;
}
}
@@ -373,8 +355,6 @@
if (((AUnionType) fieldTypes[fieldNumber]).isNullableType()) {
fieldType = ((AUnionType) fieldTypes[fieldNumber]).getNullableType();
fieldValueTypeTag = fieldType.getTypeTag();
- // fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(recordBits,
- // fieldOffsets[fieldNumber], typeTag, false);
}
} else {
fieldValueTypeTag = fieldTypes[fieldNumber].getTypeTag();
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/ExternalDataScanOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java
similarity index 96%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/ExternalDataScanOperatorDescriptor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java
index fee99d8..aed8bb9 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/ExternalDataScanOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.api.IDataSourceAdapter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java
index 59ad076..82ca715 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.external.operators;
-import java.io.File;
import java.util.List;
import org.apache.hyracks.api.context.IHyracksTaskContext;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedCollectOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
similarity index 85%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedCollectOperatorDescriptor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
index 715b68b..a929eec 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedCollectOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
@@ -16,21 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.IngestionRuntime;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedSubscriptionManager;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedSubscriptionManager;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.runtime.IngestionRuntime;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -81,12 +82,13 @@
this.subscriptionLocation = subscriptionLocation;
}
+ @Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
- this.subscriptionManager = runtimeCtx.getFeedManager().getFeedSubscriptionManager();
+ this.subscriptionManager = ((IFeedManager) runtimeCtx.getFeedManager()).getFeedSubscriptionManager();
ISubscribableRuntime sourceRuntime = null;
IOperatorNodePushable nodePushable = null;
switch (subscriptionLocation) {
@@ -96,8 +98,8 @@
FeedRuntimeType.INTAKE, partition);
sourceRuntime = getIntakeRuntime(feedSubscribableRuntimeId);
if (sourceRuntime == null) {
- throw new HyracksDataException("Source intake task not found for source feed id "
- + sourceFeedId);
+ throw new HyracksDataException(
+ "Source intake task not found for source feed id " + sourceFeedId);
}
nodePushable = new FeedCollectOperatorNodePushable(ctx, sourceFeedId, connectionId,
feedPolicyProperties, partition, nPartitions, sourceRuntime);
@@ -112,8 +114,7 @@
case SOURCE_FEED_COMPUTE_STAGE:
SubscribableFeedRuntimeId feedSubscribableRuntimeId = new SubscribableFeedRuntimeId(sourceFeedId,
FeedRuntimeType.COMPUTE, partition);
- sourceRuntime = (ISubscribableRuntime) subscriptionManager
- .getSubscribableRuntime(feedSubscribableRuntimeId);
+ sourceRuntime = subscriptionManager.getSubscribableRuntime(feedSubscribableRuntimeId);
if (sourceRuntime == null) {
throw new HyracksDataException("Source compute task not found for source feed id " + sourceFeedId
+ " " + FeedRuntimeType.COMPUTE + "[" + partition + "]");
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedCollectOperatorNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
similarity index 84%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedCollectOperatorNodePushable.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
index 8f9c8f3..8916af6 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedCollectOperatorNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
@@ -16,27 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.feeds.CollectionRuntime;
-import org.apache.asterix.common.feeds.FeedCollectRuntimeInputHandler;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedFrameCollector.State;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeInputHandler;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedOperatorOutputSideHandler;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedOperatorOutputSideHandler;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.dataflow.CollectTransformFeedFrameWriter;
+import org.apache.asterix.external.feed.dataflow.FeedCollectRuntimeInputHandler;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector.State;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.CollectionRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
@@ -76,7 +77,7 @@
policyAccessor = new FeedPolicyAccessor(feedPolicy);
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
- this.feedManager = runtimeCtx.getFeedManager();
+ this.feedManager = (IFeedManager) runtimeCtx.getFeedManager();
}
@Override
@@ -133,17 +134,16 @@
private void beginNewFeed(FeedRuntimeId runtimeId) throws Exception {
writer.open();
IFrameWriter outputSideWriter = writer;
- if (((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId()).getFeedRuntimeType().equals(
- FeedRuntimeType.COMPUTE)) {
+ if (((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId()).getFeedRuntimeType()
+ .equals(FeedRuntimeType.COMPUTE)) {
outputSideWriter = new CollectTransformFeedFrameWriter(ctx, writer, sourceRuntime, outputRecordDescriptor,
connectionId);
this.recordDesc = sourceRuntime.getRecordDescriptor();
}
FrameTupleAccessor tupleAccessor = new FrameTupleAccessor(recordDesc);
- inputSideHandler = new FeedCollectRuntimeInputHandler(ctx, connectionId, runtimeId, outputSideWriter, policyAccessor,
- false, tupleAccessor, recordDesc,
- feedManager, nPartitions);
+ inputSideHandler = new FeedCollectRuntimeInputHandler(ctx, connectionId, runtimeId, outputSideWriter,
+ policyAccessor, false, tupleAccessor, recordDesc, feedManager, nPartitions);
collectRuntime = new CollectionRuntime(connectionId, runtimeId, inputSideHandler, outputSideWriter,
sourceRuntime, feedPolicy);
@@ -177,8 +177,7 @@
outputRecordDescriptor, connectionId);
inputSideHandler = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, wrapper, policyAccessor, false,
- new FrameTupleAccessor(recordDesc), recordDesc, feedManager,
- nPartitions);
+ new FrameTupleAccessor(recordDesc), recordDesc, feedManager, nPartitions);
collectRuntime = new CollectionRuntime(connectionId, runtimeId, inputSideHandler, wrapper, sourceRuntime,
feedPolicy);
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
similarity index 84%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index 54c9af5..a18ebcd 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@ -16,21 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.util.Map;
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.IngestionRuntime;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedSubscriptionManager;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedSubscriptionManager;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.IngestionRuntime;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
import org.apache.asterix.external.library.ExternalLibraryManager;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
@@ -71,7 +72,7 @@
private ARecordType adapterOutputType;
- public FeedIntakeOperatorDescriptor(JobSpecification spec, PrimaryFeed primaryFeed, IAdapterFactory adapterFactory,
+ public FeedIntakeOperatorDescriptor(JobSpecification spec, IFeed primaryFeed, IAdapterFactory adapterFactory,
ARecordType adapterOutputType, FeedPolicyAccessor policyAccessor) {
super(spec, 0, 1);
this.feedId = new FeedId(primaryFeed.getDataverseName(), primaryFeed.getFeedName());
@@ -80,13 +81,13 @@
this.policyAccessor = policyAccessor;
}
- public FeedIntakeOperatorDescriptor(JobSpecification spec, PrimaryFeed primaryFeed, String adapterLibraryName,
+ public FeedIntakeOperatorDescriptor(JobSpecification spec, IFeed primaryFeed, String adapterLibraryName,
String adapterFactoryClassName, ARecordType adapterOutputType, FeedPolicyAccessor policyAccessor) {
super(spec, 0, 1);
this.feedId = new FeedId(primaryFeed.getDataverseName(), primaryFeed.getFeedName());
this.adaptorFactoryClassName = adapterFactoryClassName;
this.adaptorLibraryName = adapterLibraryName;
- this.adaptorConfiguration = primaryFeed.getAdaptorConfiguration();
+ this.adaptorConfiguration = primaryFeed.getAdapterConfiguration();
this.adapterOutputType = adapterOutputType;
this.policyAccessor = policyAccessor;
}
@@ -96,7 +97,8 @@
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
- IFeedSubscriptionManager feedSubscriptionManager = runtimeCtx.getFeedManager().getFeedSubscriptionManager();
+ IFeedSubscriptionManager feedSubscriptionManager = ((IFeedManager) runtimeCtx.getFeedManager())
+ .getFeedSubscriptionManager();
SubscribableFeedRuntimeId feedIngestionId = new SubscribableFeedRuntimeId(feedId, FeedRuntimeType.INTAKE,
partition);
IngestionRuntime ingestionRuntime = (IngestionRuntime) feedSubscriptionManager
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
similarity index 88%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
index 5085087..b31f2bf 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.util.ArrayList;
import java.util.HashMap;
@@ -25,21 +25,22 @@
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.feeds.CollectionRuntime;
-import org.apache.asterix.common.feeds.DistributeFeedFrameWriter;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.IngestionRuntime;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IAdapterRuntimeManager;
-import org.apache.asterix.common.feeds.api.IAdapterRuntimeManager.State;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedSubscriptionManager;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.common.feeds.api.ISubscriberRuntime;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.api.IAdapterRuntimeManager;
+import org.apache.asterix.external.api.IAdapterRuntimeManager.State;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedSubscriptionManager;
+import org.apache.asterix.external.feed.api.IIntakeProgressTracker;
+import org.apache.asterix.external.feed.api.ISubscriberRuntime;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.AdapterRuntimeManager;
+import org.apache.asterix.external.feed.runtime.CollectionRuntime;
+import org.apache.asterix.external.feed.runtime.IngestionRuntime;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
+import org.apache.asterix.external.api.IFeedAdapter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
@@ -62,7 +63,7 @@
private final IAdapterFactory adapterFactory;
private IngestionRuntime ingestionRuntime;
- private IDataSourceAdapter adapter;
+ private IFeedAdapter adapter;
private IIntakeProgressTracker tracker;
private DistributeFeedFrameWriter feedFrameWriter;
@@ -75,8 +76,8 @@
this.adapterFactory = adapterFactory;
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
- this.feedSubscriptionManager = runtimeCtx.getFeedManager().getFeedSubscriptionManager();
- this.feedManager = runtimeCtx.getFeedManager();
+ this.feedManager = (IFeedManager) runtimeCtx.getFeedManager();
+ this.feedSubscriptionManager = feedManager.getFeedSubscriptionManager();
}
@Override
@@ -85,7 +86,7 @@
try {
if (ingestionRuntime == null) {
try {
- adapter = adapterFactory.createAdapter(ctx, partition);
+ adapter = (IFeedAdapter) adapterFactory.createAdapter(ctx, partition);
//TODO: Fix record tracking
// if (adapterFactory.isRecordTrackingEnabled()) {
// tracker = adapterFactory.createIntakeProgressTracker();
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorDescriptor.java
similarity index 92%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorDescriptor.java
index a1e9917..219110f 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorDescriptor.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.api.IFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedMessage;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java
similarity index 86%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java
index 313fa1a..3cb5d64 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.util.Map;
import java.util.Map.Entry;
@@ -25,32 +25,34 @@
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.feeds.CollectionRuntime;
-import org.apache.asterix.common.feeds.DistributeFeedFrameWriter;
-import org.apache.asterix.common.feeds.FeedCollectRuntimeInputHandler;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedFrameCollector;
-import org.apache.asterix.common.feeds.FeedFrameCollector.State;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedRuntime;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeInputHandler;
-import org.apache.asterix.common.feeds.FeedRuntimeManager;
-import org.apache.asterix.common.feeds.FeedTupleCommitResponseMessage;
-import org.apache.asterix.common.feeds.IngestionRuntime;
-import org.apache.asterix.common.feeds.IntakePartitionStatistics;
-import org.apache.asterix.common.feeds.MonitoredBufferTimerTasks.MonitoredBufferStorageTimerTask;
-import org.apache.asterix.common.feeds.StorageSideMonitoredBuffer;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IAdapterRuntimeManager;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedMessage;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
-import org.apache.asterix.common.feeds.message.EndFeedMessage;
-import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.api.IAdapterRuntimeManager;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedMessage;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IIntakeProgressTracker;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.feed.dataflow.FeedCollectRuntimeInputHandler;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
+import org.apache.asterix.external.feed.dataflow.FeedFrameCollector.State;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedRuntimeManager;
+import org.apache.asterix.external.feed.message.EndFeedMessage;
+import org.apache.asterix.external.feed.message.FeedTupleCommitResponseMessage;
+import org.apache.asterix.external.feed.message.PrepareStallMessage;
+import org.apache.asterix.external.feed.message.TerminateDataFlowMessage;
+import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.feed.runtime.CollectionRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.feed.runtime.IngestionRuntime;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
+import org.apache.asterix.external.feed.watch.IntakePartitionStatistics;
+import org.apache.asterix.external.feed.watch.MonitoredBufferTimerTasks.MonitoredBufferStorageTimerTask;
+import org.apache.asterix.external.feed.watch.StorageSideMonitoredBuffer;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -59,7 +61,6 @@
/**
* Runtime for the FeedMessageOpertorDescriptor. This operator is responsible for communicating
* a feed message to the local feed manager on the host node controller.
- *
* @see FeedMessageOperatorDescriptor
* IFeedMessage
* IFeedManager
@@ -80,7 +81,7 @@
this.partition = partition;
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
- this.feedManager = runtimeCtx.getFeedManager();
+ this.feedManager = (IFeedManager) runtimeCtx.getFeedManager();
}
@Override
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaComputeNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
similarity index 88%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaComputeNodePushable.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
index f833019..80a54be 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaComputeNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -24,18 +24,18 @@
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.feeds.DistributeFeedFrameWriter;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedRuntime;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeInputHandler;
-import org.apache.asterix.common.feeds.SubscribableFeedRuntimeId;
-import org.apache.asterix.common.feeds.SubscribableRuntime;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.external.feeds.FeedPolicyEnforcer;
-import org.apache.asterix.common.feeds.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.ISubscribableRuntime;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyEnforcer;
+import org.apache.asterix.external.feed.runtime.FeedRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
+import org.apache.asterix.external.feed.runtime.SubscribableFeedRuntimeId;
+import org.apache.asterix.external.feed.runtime.SubscribableRuntime;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IActivity;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -101,11 +101,8 @@
this.partition = partition;
this.nPartitions = nPartitions;
this.connectionId = feedConnectionId;
- this.feedManager = ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
+ this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
.getApplicationObject()).getFeedManager();
- IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject();
- this.feedManager = runtimeCtx.getFeedManager();
}
@Override
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaNodePushable.java
similarity index 85%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaNodePushable.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaNodePushable.java
index 86f8750..4dae72d 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaNodePushable.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -24,14 +24,14 @@
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedRuntime;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeInputHandler;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.external.feeds.FeedPolicyEnforcer;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyEnforcer;
+import org.apache.asterix.external.feed.runtime.FeedRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IActivity;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -99,11 +99,8 @@
this.partition = partition;
this.nPartitions = nPartitions;
this.connectionId = feedConnectionId;
- this.feedManager = ((IAsterixAppRuntimeContext) (IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject()).getFeedManager();
- IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject();
- this.feedManager = runtimeCtx.getFeedManager();
+ this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
+ .getApplicationObject()).getFeedManager();
this.operandId = operationId;
}
@@ -126,10 +123,8 @@
private void initializeNewFeedRuntime(FeedRuntimeId runtimeId) throws Exception {
this.fta = new FrameTupleAccessor(recordDesc);
- this.inputSideHandler = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId,
- (AbstractUnaryInputUnaryOutputOperatorNodePushable) coreOperator,
- policyEnforcer.getFeedPolicyAccessor(), false, fta, recordDesc, feedManager,
- nPartitions);
+ this.inputSideHandler = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, coreOperator,
+ policyEnforcer.getFeedPolicyAccessor(), false, fta, recordDesc, feedManager, nPartitions);
setupBasicRuntime(inputSideHandler);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
similarity index 96%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaOperatorDescriptor.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
index 5d88a9e..9eb6c78 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
@@ -16,12 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.util.Map;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaStoreNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
similarity index 84%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaStoreNodePushable.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
index b409745..f75b3eb 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetaStoreNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.metadata.feeds;
+package org.apache.asterix.external.operators;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -25,14 +25,14 @@
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.dataflow.AsterixLSMInsertDeleteOperatorNodePushable;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedRuntime;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.FeedRuntimeInputHandler;
-import org.apache.asterix.common.feeds.api.IFeedManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
-import org.apache.asterix.external.feeds.FeedPolicyEnforcer;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyEnforcer;
+import org.apache.asterix.external.feed.runtime.FeedRuntime;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IActivity;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -98,11 +98,8 @@
this.partition = partition;
this.nPartitions = nPartitions;
this.connectionId = feedConnectionId;
- this.feedManager = ((IAsterixAppRuntimeContext) (IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject()).getFeedManager();
- IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject();
- this.feedManager = runtimeCtx.getFeedManager();
+ this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
+ .getApplicationObject()).getFeedManager();
this.operandId = operationId;
}
@@ -130,11 +127,11 @@
}
this.fta = new FrameTupleAccessor(recordDesc);
this.inputSideHandler = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, coreOperator,
- policyEnforcer.getFeedPolicyAccessor(), true, fta, recordDesc, feedManager,
- nPartitions);
- if(coreOperator instanceof AsterixLSMInsertDeleteOperatorNodePushable){
+ policyEnforcer.getFeedPolicyAccessor(), policyEnforcer.getFeedPolicyAccessor().bufferingEnabled(), fta,
+ recordDesc, feedManager, nPartitions);
+ if (coreOperator instanceof AsterixLSMInsertDeleteOperatorNodePushable) {
AsterixLSMInsertDeleteOperatorNodePushable indexOp = (AsterixLSMInsertDeleteOperatorNodePushable) coreOperator;
- if(!indexOp.isPrimary()){
+ if (!indexOp.isPrimary()) {
inputSideHandler.setBufferingEnabled(false);
}
}
@@ -149,8 +146,8 @@
this.inputSideHandler.setCoreOperator(coreOperator);
feedRuntime.setMode(Mode.PROCESS);
if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Retreived state from the zombie instance from previous execution for " + runtimeType
- + " node.");
+ LOGGER.warning(
+ "Retreived state from the zombie instance from previous execution for " + runtimeType + " node.");
}
}
@@ -158,7 +155,7 @@
coreOperator.setOutputFrameWriter(0, writer, recordDesc);
FeedRuntimeId runtimeId = new FeedRuntimeId(runtimeType, partition, operandId);
feedRuntime = new FeedRuntime(runtimeId, inputHandler, writer);
- feedManager.getFeedConnectionManager().registerFeedRuntime(connectionId, (FeedRuntime) feedRuntime);
+ feedManager.getFeedConnectionManager().registerFeedRuntime(connectionId, feedRuntime);
}
@Override
@@ -216,8 +213,7 @@
private void deregister() {
if (feedRuntime != null) {
- feedManager.getFeedConnectionManager().deRegisterFeedRuntime(connectionId,
- ((FeedRuntime) feedRuntime).getRuntimeId());
+ feedManager.getFeedConnectionManager().deRegisterFeedRuntime(connectionId, feedRuntime.getRuntimeId());
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java
index 860d35f..129b62f 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java
@@ -21,6 +21,7 @@
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.BitSet;
import java.util.List;
import java.util.Map;
@@ -1145,4 +1146,10 @@
recordBuilderPool.reset();
abvsBuilderPool.reset();
}
+
+ @Override
+ public boolean reset(InputStream in) throws IOException {
+ admLexer.reInit(new InputStreamReader(in));
+ return true;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
index 146064a..6c399c3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
@@ -198,11 +198,17 @@
}
@Override
- public void setInputStream(InputStream in) throws Exception {
+ public void setInputStream(InputStream in) throws IOException {
cursor = new FieldCursorForDelimitedDataParser(new InputStreamReader(in), fieldDelimiter, quote);
if (in != null && hasHeader) {
cursor.nextRecord();
while (cursor.nextField());
}
}
+
+ @Override
+ public boolean reset(InputStream in) throws IOException {
+ cursor = new FieldCursorForDelimitedDataParser(new InputStreamReader(in), fieldDelimiter, quote);
+ return true;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
index 649ca43..c5b39df 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
@@ -30,10 +30,6 @@
import org.apache.asterix.external.dataset.adapter.GenericAdapter;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.library.ExternalLibraryManager;
-import org.apache.asterix.external.runtime.GenericSocketFeedAdapter;
-import org.apache.asterix.external.runtime.GenericSocketFeedAdapterFactory;
-import org.apache.asterix.external.runtime.SocketClientAdapter;
-import org.apache.asterix.external.runtime.SocketClientAdapterFactory;
import org.apache.asterix.external.util.ExternalDataCompatibilityUtils;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.om.types.ARecordType;
@@ -47,16 +43,16 @@
Map<String, Class<? extends IAdapterFactory>> adapterFactories = new HashMap<String, Class<? extends IAdapterFactory>>();
// Class names
adapterFactories.put(GenericAdapter.class.getName(), GenericAdapterFactory.class);
- adapterFactories.put(GenericSocketFeedAdapter.class.getName(), GenericSocketFeedAdapterFactory.class);
- adapterFactories.put(SocketClientAdapter.class.getName(), SocketClientAdapterFactory.class);
-
// Aliases
adapterFactories.put(ExternalDataConstants.ALIAS_GENERIC_ADAPTER, GenericAdapterFactory.class);
adapterFactories.put(ExternalDataConstants.ALIAS_HDFS_ADAPTER, GenericAdapterFactory.class);
adapterFactories.put(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER, GenericAdapterFactory.class);
- adapterFactories.put(ExternalDataConstants.ALIAS_SOCKET_ADAPTER, GenericSocketFeedAdapterFactory.class);
- adapterFactories.put(ExternalDataConstants.ALIAS_SOCKET_CLIENT_ADAPTER, SocketClientAdapterFactory.class);
+ adapterFactories.put(ExternalDataConstants.ALIAS_SOCKET_ADAPTER, GenericAdapterFactory.class);
+ adapterFactories.put(ExternalDataConstants.ALIAS_SOCKET_CLIENT_ADAPTER, GenericAdapterFactory.class);
adapterFactories.put(ExternalDataConstants.ALIAS_FILE_FEED_ADAPTER, GenericAdapterFactory.class);
+ adapterFactories.put(ExternalDataConstants.ALIAS_TWITTER_PULL_ADAPTER, GenericAdapterFactory.class);
+ adapterFactories.put(ExternalDataConstants.ALIAS_TWITTER_PUSH_ADAPTER, GenericAdapterFactory.class);
+ adapterFactories.put(ExternalDataConstants.ALIAS_LOCALFS_PUSH_ADAPTER, GenericAdapterFactory.class);
// Compatability
adapterFactories.put(ExternalDataConstants.ADAPTER_HDFS_CLASSNAME, GenericAdapterFactory.class);
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
index 68a3942..dfe7aed5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
@@ -28,14 +28,19 @@
import org.apache.asterix.external.api.IInputStreamProviderFactory;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IRecordDataParserFactory;
+import org.apache.asterix.external.api.IRecordFlowController;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.api.IStreamDataParser;
import org.apache.asterix.external.api.IStreamDataParserFactory;
+import org.apache.asterix.external.api.IStreamFlowController;
+import org.apache.asterix.external.dataflow.FeedRecordDataFlowController;
+import org.apache.asterix.external.dataflow.FeedStreamDataFlowController;
import org.apache.asterix.external.dataflow.IndexingDataFlowController;
import org.apache.asterix.external.dataflow.RecordDataFlowController;
import org.apache.asterix.external.dataflow.StreamDataFlowController;
import org.apache.asterix.external.util.DataflowUtils;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -60,9 +65,11 @@
Map<String, String> configuration, boolean indexingOp) throws Exception {
switch (dataSourceFactory.getDataSourceType()) {
case RECORDS:
- RecordDataFlowController recordDataFlowController;
+ IRecordFlowController recordDataFlowController = null;
if (indexingOp) {
recordDataFlowController = new IndexingDataFlowController();
+ } else if (ExternalDataUtils.isFeed(configuration)) {
+ recordDataFlowController = new FeedRecordDataFlowController();
} else {
recordDataFlowController = new RecordDataFlowController();
}
@@ -77,7 +84,12 @@
recordDataFlowController.setRecordParser(dataParser);
return recordDataFlowController;
case STREAM:
- StreamDataFlowController streamDataFlowController = new StreamDataFlowController();
+ IStreamFlowController streamDataFlowController = null;
+ if (ExternalDataUtils.isFeed(configuration)) {
+ streamDataFlowController = new FeedStreamDataFlowController();
+ } else {
+ streamDataFlowController = new StreamDataFlowController();
+ }
streamDataFlowController.configure(configuration, ctx);
streamDataFlowController.setTupleForwarder(DataflowUtils.getTupleForwarder(configuration));
IInputStreamProviderFactory streamProviderFactory = (IInputStreamProviderFactory) dataSourceFactory;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
index c69e12c..a7ab062 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
@@ -27,6 +27,7 @@
import org.apache.asterix.external.input.HDFSDataSourceFactory;
import org.apache.asterix.external.input.record.reader.factory.LineRecordReaderFactory;
import org.apache.asterix.external.input.record.reader.factory.SemiStructuredRecordReaderFactory;
+import org.apache.asterix.external.input.record.reader.factory.TwitterRecordReaderFactory;
import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamProviderFactory;
import org.apache.asterix.external.input.stream.factory.SocketInputStreamProviderFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
@@ -92,8 +93,12 @@
.setInputStreamFactoryProvider(DatasourceFactoryProvider.getInputStreamFactory(
ExternalDataUtils.getRecordReaderStreamName(configuration), configuration));;
break;
+ case ExternalDataConstants.READER_TWITTER_PULL:
+ case ExternalDataConstants.READER_TWITTER_PUSH:
+ readerFactory = new TwitterRecordReaderFactory();
+ break;
default:
- throw new AsterixException("unknown input stream factory");
+ throw new AsterixException("unknown record reader factory");
}
}
return readerFactory;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/GenericSocketFeedAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/GenericSocketFeedAdapter.java
deleted file mode 100644
index dcf3b51..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/GenericSocketFeedAdapter.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.runtime;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.util.logging.Level;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.external.dataset.adapter.StreamBasedAdapter;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
-
-public class GenericSocketFeedAdapter extends StreamBasedAdapter {
-
- private static final long serialVersionUID = 1L;
-
- private final int port;
- private SocketFeedServer socketFeedServer;
-
- public GenericSocketFeedAdapter(ITupleParserFactory parserFactory, ARecordType outputType, int port,
- IHyracksTaskContext ctx, int partition) throws AsterixException, IOException {
- super(parserFactory, outputType, ctx, partition);
- this.port = port;
- this.socketFeedServer = new SocketFeedServer(outputType, port);
- }
-
- @Override
- public void start(int partition, IFrameWriter writer) throws Exception {
- super.start(partition, writer);
- }
-
- @Override
- public InputStream getInputStream(int partition) throws IOException {
- return socketFeedServer.getInputStream();
- }
-
- private static class SocketFeedServer {
- private ServerSocket serverSocket;
- private InputStream inputStream;
-
- public SocketFeedServer(ARecordType outputtype, int port) throws IOException, AsterixException {
- try {
- serverSocket = new ServerSocket(port);
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("port: " + port + " unusable ");
- }
- }
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Feed server configured to use port: " + port);
- }
- }
-
- public InputStream getInputStream() {
- Socket socket;
- try {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("waiting for client at " + serverSocket.getLocalPort());
- }
- socket = serverSocket.accept();
- inputStream = socket.getInputStream();
- } catch (IOException e) {
- if (LOGGER.isLoggable(Level.SEVERE)) {
- LOGGER.severe("Unable to create input stream required for feed ingestion");
- }
- }
- return inputStream;
- }
-
- public void stop() throws IOException {
- try {
- serverSocket.close();
- } catch (IOException ioe) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Unable to close socket at " + serverSocket.getLocalPort());
- }
- }
- }
-
- }
-
- @Override
- public boolean stop() throws Exception {
- socketFeedServer.stop();
- return true;
- }
-
- @Override
- public boolean handleException(Throwable e) {
- try {
- this.socketFeedServer = new SocketFeedServer((ARecordType) sourceDatatype, port);
- return true;
- } catch (Exception re) {
- return false;
- }
- }
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/GenericSocketFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/GenericSocketFeedAdapterFactory.java
deleted file mode 100644
index e19d757..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/GenericSocketFeedAdapterFactory.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.runtime;
-
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.asterix.external.api.IAdapterFactory;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.util.AsterixRuntimeUtil;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
-
-/**
- * Factory class for creating @see{GenericSocketFeedAdapter} The
- * adapter listens at a port for receiving data (from external world).
- * Data received is transformed into Asterix Data Format (ADM).
- */
-public class GenericSocketFeedAdapterFactory implements IAdapterFactory {
-
- private static final long serialVersionUID = 1L;
-
- private ARecordType outputType;
-
- private List<Pair<String, Integer>> sockets;
-
- private Mode mode = Mode.IP;
-
- private Map<String, String> configuration;
-
- private ITupleParserFactory parserFactory;
-
- public static final String KEY_SOCKETS = "sockets";
-
- public static final String KEY_MODE = "address-type";
-
- public static enum Mode {
- NC,
- IP
- }
-
- @Override
- public String getAlias() {
- return "socket_adapter";
- }
-
- public List<Pair<String, Integer>> getSockets() {
- return sockets;
- }
-
- @Override
- public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
- this.configuration = configuration;
- this.configureSockets(configuration);
- this.configureFormat(outputType);
- this.outputType = outputType;
- }
-
- private void configureFormat(ARecordType outputType2) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
- List<String> locations = new ArrayList<String>();
- for (Pair<String, Integer> socket : sockets) {
- locations.add(socket.first);
- }
- return new AlgebricksAbsolutePartitionConstraint(locations.toArray(new String[] {}));
- }
-
- @Override
- public IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
- Pair<String, Integer> socket = sockets.get(partition);
- return new GenericSocketFeedAdapter(parserFactory, outputType, socket.second, ctx, partition);
- }
-
- private void configureSockets(Map<String, String> configuration) throws Exception {
- sockets = new ArrayList<Pair<String, Integer>>();
- String modeValue = configuration.get(KEY_MODE);
- if (modeValue != null) {
- mode = Mode.valueOf(modeValue.trim().toUpperCase());
- }
- String socketsValue = configuration.get(KEY_SOCKETS);
- if (socketsValue == null) {
- throw new IllegalArgumentException("\'sockets\' parameter not specified as part of adapter configuration");
- }
- Map<InetAddress, Set<String>> ncMap = AsterixRuntimeUtil.getNodeControllerMap();
- List<String> ncs = AsterixRuntimeUtil.getAllNodeControllers();
- String[] socketsArray = socketsValue.split(",");
- Random random = new Random();
- for (String socket : socketsArray) {
- String[] socketTokens = socket.split(":");
- String host = socketTokens[0].trim();
- int port = Integer.parseInt(socketTokens[1].trim());
- Pair<String, Integer> p = null;
- switch (mode) {
- case IP:
- Set<String> ncsOnIp = ncMap.get(InetAddress.getByName(host));
- if (ncsOnIp == null || ncsOnIp.isEmpty()) {
- throw new IllegalArgumentException("Invalid host " + host
- + " as it is not part of the AsterixDB cluster. Valid choices are "
- + StringUtils.join(ncMap.keySet(), ", "));
- }
- String[] ncArray = ncsOnIp.toArray(new String[] {});
- String nc = ncArray[random.nextInt(ncArray.length)];
- p = new Pair<String, Integer>(nc, port);
- break;
-
- case NC:
- p = new Pair<String, Integer>(host, port);
- if (!ncs.contains(host)) {
- throw new IllegalArgumentException(
- "Invalid NC " + host + " as it is not part of the AsterixDB cluster. Valid choices are "
- + StringUtils.join(ncs, ", "));
-
- }
- break;
- }
- sockets.add(p);
- }
- }
-
- @Override
- public ARecordType getAdapterOutputType() {
- return outputType;
- }
-}
\ No newline at end of file
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/SocketClientAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/SocketClientAdapter.java
deleted file mode 100644
index db38c12..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/SocketClientAdapter.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.runtime;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.OutputStream;
-import java.net.Socket;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.hyracks.api.comm.IFrameWriter;
-
-public class SocketClientAdapter implements IDataSourceAdapter {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOGGER = Logger.getLogger(SocketClientAdapter.class.getName());
-
- private static final String LOCALHOST = "127.0.0.1";
-
- private static final long RECONNECT_PERIOD = 2000;
-
- private final String localFile;
-
- private final int port;
-
- private boolean continueStreaming = true;
-
- public SocketClientAdapter(Integer port, String localFile) {
- this.localFile = localFile;
- this.port = port;
- }
-
- @Override
- public void start(int partition, IFrameWriter writer) throws Exception {
- Socket socket = waitForReceiver();
- OutputStream os = socket.getOutputStream();
- FileInputStream fin = new FileInputStream(new File(localFile));
- byte[] chunk = new byte[1024];
- int read;
- try {
- while (continueStreaming) {
- read = fin.read(chunk);
- if (read > 0) {
- os.write(chunk, 0, read);
- } else {
- break;
- }
- }
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Finished streaming file " + localFile + "to port [" + port + "]");
- }
-
- } finally {
- socket.close();
- fin.close();
- }
-
- }
-
- private Socket waitForReceiver() throws Exception {
- Socket socket = null;
- while (socket == null) {
- try {
- socket = new Socket(LOCALHOST, port);
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Receiver not ready, would wait for " + (RECONNECT_PERIOD / 1000)
- + " seconds before reconnecting");
- }
- Thread.sleep(RECONNECT_PERIOD);
- }
- }
- return socket;
- }
-
- @Override
- public boolean stop() throws Exception {
- continueStreaming = false;
- return true;
- }
-
- @Override
- public boolean handleException(Throwable e) {
- return false;
- }
-
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/SocketClientAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/SocketClientAdapterFactory.java
deleted file mode 100644
index a1e90a8..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/SocketClientAdapterFactory.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.runtime;
-
-import java.util.Map;
-
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.asterix.external.api.IAdapterFactory;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-
-public class SocketClientAdapterFactory implements IAdapterFactory {
-
- private static final long serialVersionUID = 1L;
-
- private ARecordType outputType;
-
- private GenericSocketFeedAdapterFactory genericSocketAdapterFactory;
-
- private String[] fileSplits;
-
- public static final String KEY_FILE_SPLITS = "file_splits";
-
- @Override
- public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
- this.outputType = outputType;
- String fileSplitsValue = configuration.get(KEY_FILE_SPLITS);
- if (fileSplitsValue == null) {
- throw new IllegalArgumentException(
- "File splits not specified. File split is specified as a comma separated list of paths");
- }
- fileSplits = fileSplitsValue.trim().split(",");
- genericSocketAdapterFactory = new GenericSocketFeedAdapterFactory();
- genericSocketAdapterFactory.configure(configuration, outputType);
- }
-
- @Override
- public String getAlias() {
- return ExternalDataConstants.ALIAS_SOCKET_CLIENT_ADAPTER;
- }
-
- @Override
- public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
- return genericSocketAdapterFactory.getPartitionConstraint();
- }
-
- @Override
- public IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
- Pair<String, Integer> socket = genericSocketAdapterFactory.getSockets().get(partition);
- return new SocketClientAdapter(socket.second, fileSplits[partition]);
- }
-
- @Override
- public ARecordType getAdapterOutputType() {
- return outputType;
- }
-
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/DataGenerator.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java
similarity index 99%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/runtime/DataGenerator.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java
index cbf488c..d999e7e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/DataGenerator.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.runtime;
+package org.apache.asterix.external.util;
import java.nio.CharBuffer;
import java.util.ArrayList;
@@ -25,8 +25,6 @@
import java.util.List;
import java.util.Random;
-import org.apache.asterix.external.util.Datatypes;
-
public class DataGenerator {
private RandomDateGenerator randDateGen;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataflowUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataflowUtils.java
index ea13f25..90c74e1 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataflowUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataflowUtils.java
@@ -24,6 +24,7 @@
import org.apache.asterix.common.parse.ITupleForwarder;
import org.apache.asterix.common.parse.ITupleForwarder.TupleForwardPolicy;
import org.apache.asterix.external.dataflow.CounterTimerTupleForwarder;
+import org.apache.asterix.external.dataflow.FeedTupleForwarder;
import org.apache.asterix.external.dataflow.FrameFullTupleForwarder;
import org.apache.asterix.external.dataflow.RateControlledTupleForwarder;
import org.apache.hyracks.api.comm.IFrameWriter;
@@ -37,7 +38,7 @@
if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
appender.flush(writer, true);
if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
- throw new IllegalStateException();
+ throw new HyracksDataException("Tuple is too large for a frame");
}
}
}
@@ -46,12 +47,18 @@
ITupleForwarder policy = null;
ITupleForwarder.TupleForwardPolicy policyType = null;
String propValue = configuration.get(ITupleForwarder.FORWARD_POLICY);
- if (propValue == null) {
+ if (ExternalDataUtils.isFeed(configuration)) {
+ //TODO pass this value in the configuration and avoid this check for feeds
+ policyType = TupleForwardPolicy.FEED;
+ } else if (propValue == null) {
policyType = TupleForwardPolicy.FRAME_FULL;
} else {
policyType = TupleForwardPolicy.valueOf(propValue.trim().toUpperCase());
}
switch (policyType) {
+ case FEED:
+ policy = new FeedTupleForwarder();
+ break;
case FRAME_FULL:
policy = new FrameFullTupleForwarder();
break;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
index a544638..17b194c 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
@@ -22,7 +22,6 @@
/*
The following assumes this DDL (but ignoring the field name orders):
-
create type TwitterUser if not exists as open{
screen_name: string,
language: string,
@@ -31,7 +30,6 @@
name: string,
followers_count: int32
};
-
create type Tweet if not exists as open{
id: string,
user: TwitterUser,
@@ -40,7 +38,6 @@
created_at:string,
message_text:string
};
-
*/
public static class Tweet {
public static final String ID = "id";
@@ -62,10 +59,8 @@
}
-
/*
The following assumes this DDL (but ignoring the field name orders):
-
create type ProcessedTweet if not exists as open {
id: string,
user_name:string,
@@ -75,7 +70,6 @@
country: string,
topics: [string]
};
-
*/
public static final class ProcessedTweet {
public static final String USER_NAME = "user_name";
@@ -83,5 +77,4 @@
public static final String TOPICS = "topics";
}
-
-}
+}
\ No newline at end of file
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
index 7f91a2b..4e6401a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
@@ -49,10 +49,11 @@
}
//TODO:Add remaining aliases
- public static void addCompatabilityParameters(String adapterClassname, ARecordType itemType,
+ public static void addCompatabilityParameters(String adapterName, ARecordType itemType,
Map<String, String> configuration) throws AsterixException {
- if (adapterClassname.equals(ExternalDataConstants.ALIAS_HDFS_ADAPTER)
- || adapterClassname.equalsIgnoreCase(ExternalDataConstants.ADAPTER_HDFS_CLASSNAME)) {
+ // HDFS
+ if (adapterName.equals(ExternalDataConstants.ALIAS_HDFS_ADAPTER)
+ || adapterName.equalsIgnoreCase(ExternalDataConstants.ADAPTER_HDFS_CLASSNAME)) {
if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
throw new AsterixException("Unspecified format parameter for HDFS adapter");
}
@@ -65,21 +66,45 @@
configuration.put(ExternalDataConstants.KEY_READER_STREAM, ExternalDataConstants.ALIAS_HDFS_ADAPTER);
}
}
- if (adapterClassname.equals(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER)
- || adapterClassname.contains(ExternalDataConstants.ADAPTER_LOCALFS_CLASSNAME)) {
+
+ // Local Filesystem
+ if (adapterName.equals(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER)
+ || adapterName.contains(ExternalDataConstants.ADAPTER_LOCALFS_CLASSNAME)
+ || adapterName.contains(ExternalDataConstants.ALIAS_LOCALFS_PUSH_ADAPTER)) {
if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
throw new AsterixException("Unspecified format parameter for local file system adapter");
}
configuration.put(ExternalDataConstants.KEY_READER, configuration.get(ExternalDataConstants.KEY_FORMAT));
configuration.put(ExternalDataConstants.KEY_READER_STREAM, ExternalDataConstants.ALIAS_LOCALFS_ADAPTER);
}
+
+ // Twitter (Pull)
+ if (adapterName.equals(ExternalDataConstants.ALIAS_TWITTER_PULL_ADAPTER)) {
+ configuration.put(ExternalDataConstants.KEY_READER, ExternalDataConstants.READER_TWITTER_PULL);
+ configuration.put(ExternalDataConstants.KEY_PULL, ExternalDataConstants.TRUE);
+ ExternalDataUtils.setRecordFormat(configuration, ExternalDataConstants.FORMAT_TWEET);
+ }
+
+ // Twitter (Push)
+ if (adapterName.equals(ExternalDataConstants.ALIAS_TWITTER_PUSH_ADAPTER)) {
+ configuration.put(ExternalDataConstants.KEY_READER, ExternalDataConstants.READER_TWITTER_PUSH);
+ configuration.put(ExternalDataConstants.KEY_PUSH, ExternalDataConstants.TRUE);
+ ExternalDataUtils.setRecordFormat(configuration, ExternalDataConstants.FORMAT_TWEET);
+ }
+
+ // Hive Parser
if (configuration.get(ExternalDataConstants.KEY_PARSER) != null
&& configuration.get(ExternalDataConstants.KEY_PARSER).equals(ExternalDataConstants.PARSER_HIVE)) {
configuration.put(ExternalDataConstants.KEY_PARSER, ExternalDataConstants.FORMAT_HIVE);
}
+
+ // FileSystem for Feed adapter
if (configuration.get(ExternalDataConstants.KEY_FILESYSTEM) != null) {
configuration.put(ExternalDataConstants.KEY_STREAM,
configuration.get(ExternalDataConstants.KEY_FILESYSTEM));
+ if (adapterName.equalsIgnoreCase(ExternalDataConstants.ALIAS_FILE_FEED_ADAPTER)) {
+ configuration.put(ExternalDataConstants.KEY_WAIT_FOR_DATA, ExternalDataConstants.FALSE);
+ }
}
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 2050e6a..fb2688f 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -24,7 +24,7 @@
import org.apache.hadoop.mapred.TextInputFormat;
public class ExternalDataConstants {
- //TODO: Remove unused variables.
+ // TODO: Remove unused variables.
/**
* Keys
*/
@@ -70,6 +70,9 @@
public static final String KEY_INTERVAL = "interval";
public static final String KEY_PULL = "pull";
public static final String KEY_PUSH = "push";
+ public static final String KEY_IS_FEED = "is-feed";
+ public static final String KEY_WAIT_FOR_DATA = "wait-for-data";
+ public static final String KEY_FEED_NAME = "feed";
/**
* HDFS class names
*/
@@ -94,6 +97,8 @@
public static final String READER_ADM = "adm";
public static final String READER_SEMISTRUCTURED = "semi-structured";
public static final String READER_DELIMITED = "delimited-text";
+ public static final String READER_TWITTER_PUSH = "twitter-push";
+ public static final String READER_TWITTER_PULL = "twitter-pull";
public static final String CLUSTER_LOCATIONS = "cluster-locations";
public static final String SCHEDULER = "hdfs-scheduler";
@@ -128,6 +133,7 @@
*/
public static final String ALIAS_GENERIC_ADAPTER = "adapter";
public static final String ALIAS_LOCALFS_ADAPTER = "localfs";
+ public static final String ALIAS_LOCALFS_PUSH_ADAPTER = "push_localfs";
public static final String ALIAS_HDFS_ADAPTER = "hdfs";
public static final String ALIAS_SOCKET_ADAPTER = "socket_adapter";
public static final String ALIAS_TWITTER_FIREHOSE_ADAPTER = "twitter_firehose";
@@ -136,7 +142,6 @@
public static final String ALIAS_FILE_FEED_ADAPTER = "file_feed";
public static final String ALIAS_TWITTER_PUSH_ADAPTER = "push_twitter";
public static final String ALIAS_TWITTER_PULL_ADAPTER = "pull_twitter";
- public static final String ALIAS_TWITTER_AZURE_ADAPTER = "azure_twitter";
public static final String ALIAS_CNN_ADAPTER = "cnn_feed";
/**
@@ -146,6 +151,12 @@
public static final String ADAPTER_HDFS_CLASSNAME = "org.apache.asterix.external.dataset.adapter.HDFSAdapter";
/**
+ * Constant String values
+ */
+ public static final String TRUE = "true";
+ public static final String FALSE = "false";
+
+ /**
* Constant characters
*/
public static final char ESCAPE = '\\';
@@ -160,7 +171,7 @@
/**
* Constant byte characters
*/
- public static final byte EOL = '\n';
+ public static final byte BYTE_LF = '\n';
public static final byte BYTE_CR = '\r';
/**
* Size default values
@@ -172,5 +183,4 @@
* Expected parameter values
*/
public static final String PARAMETER_OF_SIZE_ONE = "Value of size 1";
-
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataExceptionUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataExceptionUtils.java
index 9dcaef4..f16e24b 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataExceptionUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataExceptionUtils.java
@@ -18,6 +18,10 @@
*/
package org.apache.asterix.external.util;
+import java.util.Arrays;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public class ExternalDataExceptionUtils {
public static final String INCORRECT_PARAMETER = "Incorrect parameter.\n";
public static final String MISSING_PARAMETER = "Missing parameter.\n";
@@ -29,4 +33,22 @@
return INCORRECT_PARAMETER + PARAMETER_NAME + parameterName + ExternalDataConstants.LF + EXPECTED_VALUE
+ expectedValue + ExternalDataConstants.LF + PASSED_VALUE + passedValue;
}
+
+ public static String concat(String... vals) {
+ return Arrays.toString(vals);
+ }
+
+ // For now, we are accepting all exceptions as resolvable by adapter.
+ public static boolean isResolvable(Exception e) {
+ return true;
+ }
+
+ public static HyracksDataException suppress(HyracksDataException hde, Throwable th) {
+ if (hde == null) {
+ return new HyracksDataException(th);
+ } else {
+ hde.addSuppressed(th);
+ return hde;
+ }
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 7c1c1b5..c9be872 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -23,7 +23,6 @@
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedConstants;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
import org.apache.asterix.external.api.IInputStreamProviderFactory;
@@ -134,6 +133,15 @@
return parserFormat != null ? parserFormat : configuration.get(ExternalDataConstants.KEY_FORMAT);
}
+ public static void setRecordFormat(Map<String, String> configuration, String format) {
+ if (!configuration.containsKey(ExternalDataConstants.KEY_DATA_PARSER)) {
+ configuration.put(ExternalDataConstants.KEY_DATA_PARSER, format);
+ }
+ if (!configuration.containsKey(ExternalDataConstants.KEY_FORMAT)) {
+ configuration.put(ExternalDataConstants.KEY_FORMAT, format);
+ }
+ }
+
private static Map<ATypeTag, IValueParserFactory> valueParserFactoryMap = initializeValueParserFactoryMap();
private static Map<ATypeTag, IValueParserFactory> initializeValueParserFactoryMap() {
@@ -219,4 +227,31 @@
.substring(parserFactoryName.indexOf(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR) + 1))
.newInstance();
}
+
+ public static boolean isFeed(Map<String, String> configuration) {
+ if (!configuration.containsKey(ExternalDataConstants.KEY_IS_FEED)) {
+ return false;
+ } else {
+ return Boolean.parseBoolean(configuration.get(ExternalDataConstants.KEY_IS_FEED));
+ }
+ }
+
+ public static void prepareFeed(Map<String, String> configuration, String dataverseName, String feedName) {
+ if (!configuration.containsKey(ExternalDataConstants.KEY_IS_FEED)) {
+ configuration.put(ExternalDataConstants.KEY_IS_FEED, ExternalDataConstants.TRUE);
+ }
+ configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataverseName);
+ configuration.put(ExternalDataConstants.KEY_FEED_NAME, feedName);
+ }
+
+ public static boolean keepDataSourceOpen(Map<String, String> configuration) {
+ if (!configuration.containsKey(ExternalDataConstants.KEY_WAIT_FOR_DATA)) {
+ return true;
+ }
+ return Boolean.parseBoolean(configuration.get(ExternalDataConstants.KEY_WAIT_FOR_DATA));
+ }
+
+ public static String getFeedName(Map<String, String> configuration) {
+ return configuration.get(ExternalDataConstants.KEY_FEED_NAME);
+ }
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConstants.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java
similarity index 90%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConstants.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java
index 05e554b..cc21360 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedConstants.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java
@@ -16,10 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.util;
public class FeedConstants {
+ public final static String FEEDS_METADATA_DV = "feeds_metadata";
+ public final static String FAILED_TUPLE_DATASET = "failed_tuple";
+ public final static String FAILED_TUPLE_DATASET_TYPE = "FailedTupleType";
+ public final static String FAILED_TUPLE_DATASET_KEY = "id";
+
public static final class StatisticsConstants {
public static final String INTAKE_TUPLEID = "intake-tupleid";
public static final String INTAKE_PARTITION = "intake-partition";
@@ -58,7 +63,7 @@
public static final String MAX_WINDOW_ACKED = "max-window-acked";
public static final String BASE = "base";
public static final String NOT_APPLICABLE = "N/A";
-
+
}
public static final class NamingConstants {
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameUtil.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedFrameUtil.java
similarity index 87%
rename from asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameUtil.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedFrameUtil.java
index baf7a7c..a2bdd64 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/FeedFrameUtil.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedFrameUtil.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.feeds;
+package org.apache.asterix.external.util;
import java.nio.ByteBuffer;
import java.util.BitSet;
@@ -30,20 +30,22 @@
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
public class FeedFrameUtil {
-
- public static ByteBuffer getSlicedFrame(IHyracksTaskContext ctx, int tupleIndex, FrameTupleAccessor fta) throws HyracksDataException {
+ public static ByteBuffer removeBadTuple(IHyracksTaskContext ctx, int tupleIndex, FrameTupleAccessor fta)
+ throws HyracksDataException {
FrameTupleAppender appender = new FrameTupleAppender();
IFrame slicedFrame = new VSizeFrame(ctx);
appender.reset(slicedFrame, true);
- int startTupleIndex = tupleIndex + 1;
int totalTuples = fta.getTupleCount();
- for (int ti = startTupleIndex; ti < totalTuples; ti++) {
- appender.append(fta, ti);
+ for (int ti = 0; ti < totalTuples; ti++) {
+ if (ti != tupleIndex) {
+ appender.append(fta, ti);
+ }
}
return slicedFrame.getBuffer();
}
- public static ByteBuffer getSampledFrame(IHyracksTaskContext ctx, FrameTupleAccessor fta, int sampleSize) throws HyracksDataException {
+ public static ByteBuffer getSampledFrame(IHyracksTaskContext ctx, FrameTupleAccessor fta, int sampleSize)
+ throws HyracksDataException {
NChooseKIterator it = new NChooseKIterator(fta.getTupleCount(), sampleSize);
FrameTupleAppender appender = new FrameTupleAppender();
IFrame sampledFrame = new VSizeFrame(ctx);
@@ -55,8 +57,6 @@
}
return sampledFrame.getBuffer();
}
-
-
private static class NChooseKIterator {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
new file mode 100644
index 0000000..72b438d
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.TreeSet;
+
+import org.apache.commons.io.FileUtils;
+
+public class FeedLogManager {
+
+ public enum LogEntryType {
+ START, // partition start
+ END, // partition end
+ COMMIT, // a record commit within a partition
+ SNAPSHOT // an identifier that partitions with identifiers before this one should be ignored
+ }
+
+ public static final String PROGRESS_LOG_FILE_NAME = "progress.log";
+ public static final String ERROR_LOG_FILE_NAME = "error.log";
+ public static final String BAD_RECORDS_FILE_NAME = "failed_record.log";
+ public static final String START_PREFIX = "s:";
+ public static final String END_PREFIX = "e:";
+ public static final int PREFIX_SIZE = 2;
+ private String currentPartition;
+ private TreeSet<String> completed;
+ private Path dir;
+ private BufferedWriter progressLogger;
+ private BufferedWriter errorLogger;
+ private BufferedWriter recordLogger;
+
+ public FeedLogManager(File file) {
+ this.dir = file.toPath();
+ this.completed = new TreeSet<String>();
+ }
+
+ public void endPartition() throws IOException {
+ logProgress(END_PREFIX + currentPartition);
+ completed.add(currentPartition);
+ }
+
+ public void endPartition(String partition) throws IOException {
+ currentPartition = partition;
+ logProgress(END_PREFIX + currentPartition);
+ completed.add(currentPartition);
+ }
+
+ public void startPartition(String partition) throws IOException {
+ currentPartition = partition;
+ logProgress(START_PREFIX + currentPartition);
+ }
+
+ public boolean exists() {
+ return Files.exists(dir);
+ }
+
+ public void open() throws IOException {
+ // read content of logs.
+ BufferedReader reader = Files.newBufferedReader(
+ Paths.get(dir.toAbsolutePath().toString() + File.separator + PROGRESS_LOG_FILE_NAME));
+ String log = reader.readLine();
+ while (log != null) {
+ if (log.startsWith(END_PREFIX)) {
+ completed.add(getSplitId(log));
+ }
+ log = reader.readLine();
+ }
+ reader.close();
+
+ progressLogger = Files.newBufferedWriter(
+ Paths.get(dir.toAbsolutePath().toString() + File.separator + PROGRESS_LOG_FILE_NAME),
+ StandardCharsets.UTF_8, StandardOpenOption.APPEND);
+ errorLogger = Files.newBufferedWriter(
+ Paths.get(dir.toAbsolutePath().toString() + File.separator + ERROR_LOG_FILE_NAME),
+ StandardCharsets.UTF_8, StandardOpenOption.APPEND);
+ recordLogger = Files.newBufferedWriter(
+ Paths.get(dir.toAbsolutePath().toString() + File.separator + BAD_RECORDS_FILE_NAME),
+ StandardCharsets.UTF_8, StandardOpenOption.APPEND);
+ }
+
+ public void close() throws IOException {
+ progressLogger.close();
+ errorLogger.close();
+ recordLogger.close();
+ }
+
+ public boolean create() throws IOException {
+ File f = dir.toFile();
+ f.mkdirs();
+ new File(f, PROGRESS_LOG_FILE_NAME).createNewFile();
+ new File(f, ERROR_LOG_FILE_NAME).createNewFile();
+ new File(f, BAD_RECORDS_FILE_NAME).createNewFile();
+ return true;
+ }
+
+ public boolean destroy() throws IOException {
+ File f = dir.toFile();
+ FileUtils.deleteDirectory(f);
+ return true;
+ }
+
+ public void logProgress(String log) throws IOException {
+ progressLogger.write(log);
+ progressLogger.newLine();
+ }
+
+ public void logError(String error, Throwable th) throws IOException {
+ errorLogger.append(error);
+ errorLogger.newLine();
+ errorLogger.append(th.toString());
+ errorLogger.newLine();
+ }
+
+ public void logRecord(String record, Exception e) throws IOException {
+ recordLogger.append(record);
+ recordLogger.newLine();
+ recordLogger.append(e.toString());
+ recordLogger.newLine();
+ }
+
+ public static String getSplitId(String log) {
+ return log.substring(PREFIX_SIZE);
+ }
+
+ public boolean isSplitRead(String split) {
+ return completed.contains(split);
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
new file mode 100644
index 0000000..224ee31
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.asterix.common.utils.StoragePathUtil;
+import org.apache.asterix.om.util.AsterixClusterProperties;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint.PartitionConstraintType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
+
+public class FeedUtils {
+ private static String prepareDataverseFeedName(String dataverseName, String feedName) {
+ return dataverseName + File.separator + feedName;
+ }
+
+ public static FileSplit[] splitsForAdapter(String dataverseName, String feedName,
+ AlgebricksPartitionConstraint partitionConstraints) throws Exception {
+ File relPathFile = new File(prepareDataverseFeedName(dataverseName, feedName));
+ if (partitionConstraints.getPartitionConstraintType() == PartitionConstraintType.COUNT) {
+ throw new AlgebricksException("Can't create file splits for adapter with count partitioning constraints");
+ }
+ String[] locations = ((AlgebricksAbsolutePartitionConstraint) partitionConstraints).getLocations();
+ List<FileSplit> splits = new ArrayList<FileSplit>();
+ String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
+ int i = 0;
+ for (String nd : locations) {
+ // Always get the first partition
+ ClusterPartition nodePartition = AsterixClusterProperties.INSTANCE.getNodePartitions(nd)[0];
+ String storagePartitionPath = StoragePathUtil.prepareStoragePartitionPath(storageDirName,
+ nodePartition.getPartitionId());
+ // format: 'storage dir name'/partition_#/dataverse/feed/adapter_#
+ File f = new File(storagePartitionPath + File.separator + relPathFile + File.separator
+ + StoragePathUtil.ADAPTER_INSTANCE_PREFIX + i);
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartition, f));
+ i++;
+ }
+ return splits.toArray(new FileSplit[] {});
+ }
+
+ public static FileReference getAbsoluteFileRef(String relativePath, int ioDeviceId, IIOManager ioManager) {
+ return ioManager.getAbsoluteFileRef(ioDeviceId, relativePath);
+ }
+
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
new file mode 100644
index 0000000..4bb9d92
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.ClosedWatchServiceException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardWatchEventKinds;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchEvent.Kind;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+public class FileSystemWatcher {
+
+ private static Logger LOGGER = Logger.getLogger(FileSystemWatcher.class.getName());
+ private final WatchService watcher;
+ private final HashMap<WatchKey, Path> keys;
+ private final LinkedList<File> files = new LinkedList<File>();
+ private Iterator<File> it;
+ private final String expression;
+ private final FeedLogManager logManager;
+ private final Path path;
+ private final boolean isFeed;
+ private boolean done;
+ private File current;
+
+ public FileSystemWatcher(FeedLogManager logManager, Path inputResource, String expression, boolean isFeed)
+ throws IOException {
+ this.watcher = isFeed ? FileSystems.getDefault().newWatchService() : null;
+ this.keys = isFeed ? new HashMap<WatchKey, Path>() : null;
+ this.logManager = logManager;
+ this.expression = expression;
+ this.path = inputResource;
+ this.isFeed = isFeed;
+ }
+
+ public void init() throws IOException {
+ LinkedList<Path> dirs = null;
+ dirs = new LinkedList<Path>();
+ LocalFileSystemUtils.traverse(files, path.toFile(), expression, dirs);
+ it = files.iterator();
+ if (isFeed) {
+ for (Path path : dirs) {
+ register(path);
+ }
+ resume();
+ }
+ }
+
+ /**
+ * Register the given directory, and all its sub-directories, with the
+ * WatchService.
+ */
+ private void register(Path dir) throws IOException {
+ WatchKey key = dir.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_DELETE,
+ StandardWatchEventKinds.ENTRY_MODIFY);
+ keys.put(key, dir);
+ }
+
+ private void resume() throws IOException {
+ if (logManager == null) {
+ return;
+ }
+ if (logManager.exists()) {
+ logManager.open();
+ } else {
+ logManager.create();
+ logManager.open();
+ return;
+ }
+ /*
+ * Done processing the progress log file. We now have:
+ * the files that were completed.
+ */
+
+ if (it == null) {
+ return;
+ }
+ while (it.hasNext()) {
+ File file = it.next();
+ if (logManager.isSplitRead(file.getAbsolutePath())) {
+ // File was read completely, remove it from the files list
+ it.remove();
+ }
+ }
+ // reset the iterator
+ it = files.iterator();
+ }
+
+ @SuppressWarnings("unchecked")
+ static <T> WatchEvent<T> cast(WatchEvent<?> event) {
+ return (WatchEvent<T>) event;
+ }
+
+ private void handleEvents(WatchKey key) {
+ // get dir associated with the key
+ Path dir = keys.get(key);
+ if (dir == null) {
+ // This should never happen
+ if (LOGGER.isEnabledFor(Level.WARN)) {
+ LOGGER.warn("WatchKey not recognized!!");
+ }
+ return;
+ }
+ for (WatchEvent<?> event : key.pollEvents()) {
+ Kind<?> kind = event.kind();
+ // TODO: Do something about overflow events
+ // An overflow event means that some events were dropped
+ if (kind == StandardWatchEventKinds.OVERFLOW) {
+ if (LOGGER.isEnabledFor(Level.WARN)) {
+ LOGGER.warn("Overflow event. Some events might have been missed");
+ }
+ continue;
+ }
+
+ // Context for directory entry event is the file name of entry
+ WatchEvent<Path> ev = cast(event);
+ Path name = ev.context();
+ Path child = dir.resolve(name);
+ // if directory is created then register it and its sub-directories
+ if ((kind == StandardWatchEventKinds.ENTRY_CREATE)) {
+ try {
+ if (Files.isDirectory(child, LinkOption.NOFOLLOW_LINKS)) {
+ register(child);
+ } else {
+ // it is a file, add it to the files list.
+ LocalFileSystemUtils.validateAndAdd(child, expression, files);
+ }
+ } catch (IOException e) {
+ if (LOGGER.isEnabledFor(Level.ERROR)) {
+ LOGGER.error(e);
+ }
+ }
+ }
+ }
+ }
+
+ public void close() throws IOException {
+ if (!done) {
+ if (watcher != null) {
+ watcher.close();
+ }
+ if (logManager != null) {
+ if (current != null) {
+ logManager.startPartition(current.getAbsolutePath());
+ logManager.endPartition();
+ }
+ logManager.close();
+ current = null;
+ }
+ done = true;
+ }
+ }
+
+ public File next() throws IOException {
+ if (current != null && logManager != null) {
+ logManager.startPartition(current.getAbsolutePath());
+ logManager.endPartition();
+ }
+ current = it.next();
+ return current;
+ }
+
+ private boolean endOfEvents(WatchKey key) {
+ // reset key and remove from set if directory no longer accessible
+ if (!key.reset()) {
+ keys.remove(key);
+ if (keys.isEmpty()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean hasNext() {
+ if (it.hasNext()) {
+ return true;
+ }
+ if (done || !isFeed) {
+ return false;
+ }
+ files.clear();
+ // Read new Events (Polling first to add all available files)
+ WatchKey key;
+ key = watcher.poll();
+ while (key != null) {
+ handleEvents(key);
+ if (endOfEvents(key)) {
+ return false;
+ }
+ key = watcher.poll();
+ }
+ // No file was found, wait for the filesystem to push events
+ while (files.isEmpty()) {
+ try {
+ key = watcher.take();
+ } catch (InterruptedException x) {
+ if (LOGGER.isEnabledFor(Level.WARN)) {
+ LOGGER.warn("Feed Closed");
+ }
+ return false;
+ } catch (ClosedWatchServiceException e) {
+ if (LOGGER.isEnabledFor(Level.WARN)) {
+ LOGGER.warn("The watcher has exited");
+ }
+ return false;
+ }
+ handleEvents(key);
+ if (endOfEvents(key)) {
+ return false;
+ }
+ }
+ // files were found, re-create the iterator and move it one step
+ it = files.iterator();
+ return it.hasNext();
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/LocalFileSystemUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/LocalFileSystemUtils.java
new file mode 100644
index 0000000..d6e9463
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/LocalFileSystemUtils.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.LinkedList;
+import java.util.regex.Pattern;
+
+public class LocalFileSystemUtils {
+
+ //TODO: replace this method by FileUtils.iterateFilesAndDirs(.)
+ public static void traverse(final LinkedList<File> files, File root, final String expression,
+ final LinkedList<Path> dirs) throws IOException {
+ if (!Files.exists(root.toPath())) {
+ return;
+ }
+ if (!Files.isDirectory(root.toPath())) {
+ validateAndAdd(root.toPath(), expression, files);
+ }
+ //FileUtils.iterateFilesAndDirs(directory, fileFilter, dirFilter)
+ Files.walkFileTree(root.toPath(), new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) throws IOException {
+ if (!Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
+ return FileVisitResult.TERMINATE;
+ }
+ if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) {
+ if (dirs != null) {
+ dirs.add(path);
+ }
+ //get immediate children files
+ File[] content = path.toFile().listFiles();
+ for (File file : content) {
+ if (!file.isDirectory()) {
+ validateAndAdd(file.toPath(), expression, files);
+ }
+ }
+ } else {
+ // Path is a file, add to list of files if it matches the expression
+ validateAndAdd(path, expression, files);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ public static void validateAndAdd(Path path, String expression, LinkedList<File> files) {
+ if (expression == null || Pattern.matches(expression, path.toString())) {
+ files.add(new File(path.toString()));
+ }
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/DNSResolverFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolverFactory.java
similarity index 94%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/util/DNSResolverFactory.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolverFactory.java
index f8585bb..b62dda8 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/DNSResolverFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolverFactory.java
@@ -24,7 +24,7 @@
/**
* Factory for creating instance of {@link NodeResolver}
*/
-public class DNSResolverFactory implements INodeResolverFactory {
+public class NodeResolverFactory implements INodeResolverFactory {
private static final INodeResolver INSTANCE = new NodeResolver();
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/TweetGenerator.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java
similarity index 94%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/runtime/TweetGenerator.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java
index b5fd454..ec866be 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/runtime/TweetGenerator.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.runtime;
+package org.apache.asterix.external.util;
import java.io.IOException;
import java.io.OutputStream;
@@ -27,9 +27,9 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.external.runtime.DataGenerator.InitializationInfo;
-import org.apache.asterix.external.runtime.DataGenerator.TweetMessage;
-import org.apache.asterix.external.runtime.DataGenerator.TweetMessageIterator;
+import org.apache.asterix.external.util.DataGenerator.InitializationInfo;
+import org.apache.asterix.external.util.DataGenerator.TweetMessage;
+import org.apache.asterix.external.util.DataGenerator.TweetMessageIterator;
public class TweetGenerator {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetProcessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetProcessor.java
deleted file mode 100644
index f8914a6..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetProcessor.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.util;
-
-import org.apache.asterix.external.library.java.JObjectUtil;
-import org.apache.asterix.external.util.Datatypes.Tweet;
-import org.apache.asterix.om.base.AMutableDouble;
-import org.apache.asterix.om.base.AMutableInt32;
-import org.apache.asterix.om.base.AMutableRecord;
-import org.apache.asterix.om.base.AMutableString;
-import org.apache.asterix.om.base.IAObject;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.IAType;
-import twitter4j.Status;
-import twitter4j.User;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class TweetProcessor {
-
- private IAObject[] mutableTweetFields;
- private IAObject[] mutableUserFields;
- private AMutableRecord mutableRecord;
- private AMutableRecord mutableUser;
-
- private final Map<String, Integer> userFieldNameMap = new HashMap<>();
- private final Map<String, Integer> tweetFieldNameMap = new HashMap<>();
-
-
- public TweetProcessor(ARecordType recordType) {
- initFieldNames(recordType);
- mutableUserFields = new IAObject[] { new AMutableString(null), new AMutableString(null), new AMutableInt32(0),
- new AMutableInt32(0), new AMutableString(null), new AMutableInt32(0) };
- mutableUser = new AMutableRecord((ARecordType) recordType.getFieldTypes()[tweetFieldNameMap.get(Tweet.USER)], mutableUserFields);
-
- mutableTweetFields = new IAObject[] { new AMutableString(null), mutableUser, new AMutableDouble(0),
- new AMutableDouble(0), new AMutableString(null), new AMutableString(null) };
- mutableRecord = new AMutableRecord(recordType, mutableTweetFields);
-
- }
-
- // Initialize the hashmap values for the field names and positions
- private void initFieldNames(ARecordType recordType) {
- String tweetFields[] = recordType.getFieldNames();
- for (int i=0; i<tweetFields.length; i++) {
- tweetFieldNameMap.put(tweetFields[i], i);
- if (tweetFields[i].equals(Tweet.USER)) {
- IAType fieldType = recordType.getFieldTypes()[i];
- if (fieldType.getTypeTag() == ATypeTag.RECORD) {
- String userFields[] = ((ARecordType)fieldType).getFieldNames();
- for (int j=0; j<userFields.length; j++) {
- userFieldNameMap.put(userFields[j], j);
- }
- }
-
- }
- }
- }
-
-
- public AMutableRecord processNextTweet(Status tweet) {
- User user = tweet.getUser();
-
- // Tweet user data
- ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.SCREEN_NAME)]).setValue(JObjectUtil.getNormalizedString(user.getScreenName()));
- ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.LANGUAGE)]).setValue(JObjectUtil.getNormalizedString(user.getLang()));
- ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FRIENDS_COUNT)]).setValue(user.getFriendsCount());
- ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.STATUS_COUNT)]).setValue(user.getStatusesCount());
- ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.NAME)]).setValue(JObjectUtil.getNormalizedString(user.getName()));
- ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FOLLOWERS_COUNT)]).setValue(user.getFollowersCount());
-
-
- // Tweet data
- ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.ID)]).setValue(String.valueOf(tweet.getId()));
-
- int userPos = tweetFieldNameMap.get(Tweet.USER);
- for (int i = 0; i < mutableUserFields.length; i++) {
- ((AMutableRecord) mutableTweetFields[userPos]).setValueAtPos(i, mutableUserFields[i]);
- }
- if (tweet.getGeoLocation() != null) {
- ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]).setValue(tweet.getGeoLocation().getLatitude());
- ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]).setValue(tweet.getGeoLocation().getLongitude());
- } else {
- ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]).setValue(0);
- ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]).setValue(0);
- }
- ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.CREATED_AT)]).setValue(JObjectUtil.getNormalizedString(
- tweet.getCreatedAt().toString()));
- ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.MESSAGE)]).setValue(JObjectUtil.getNormalizedString(tweet.getText()));
-
- for (int i = 0; i < mutableTweetFields.length; i++) {
- mutableRecord.setValueAtPos(i, mutableTweetFields[i]);
- }
-
- return mutableRecord;
-
- }
-
- public AMutableRecord getMutableRecord() {
- return mutableRecord;
- }
-
-}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java
index 70bd3e1..16f8b0a 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java
@@ -18,14 +18,11 @@
*/
package org.apache.asterix.external.library;
-import java.util.Random;
-
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionHelper;
import org.apache.asterix.external.library.java.JObjects.JInt;
import org.apache.asterix.external.library.java.JObjects.JRecord;
import org.apache.asterix.external.library.java.JObjects.JString;
-import org.apache.asterix.external.api.IExternalScalarFunction;
-import org.apache.asterix.external.api.IFunctionHelper;
-import org.apache.asterix.external.library.java.JTypeTag;
/**
* Accepts an input record of type Open{ id: int32, text: string }
@@ -35,11 +32,8 @@
*/
public class UpperCaseFunction implements IExternalScalarFunction {
- private Random random;
-
@Override
public void initialize(IFunctionHelper functionHelper) {
- random = new Random();
}
@Override
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapter.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapter.java
index df0fb94..2fc289b 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapter.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapter.java
@@ -19,21 +19,24 @@
package org.apache.asterix.external.library.adapter;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
-import org.apache.asterix.external.dataset.adapter.StreamBasedAdapter;
+import org.apache.asterix.external.api.IFeedAdapter;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.dataflow.std.file.ITupleParser;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
-public class TestTypedAdapter extends StreamBasedAdapter {
+public class TestTypedAdapter implements IFeedAdapter {
private static final long serialVersionUID = 1L;
@@ -45,25 +48,34 @@
private DummyGenerator generator;
+ protected final ITupleParser tupleParser;
+
+ protected final IAType sourceDatatype;
+
+ protected static final Logger LOGGER = Logger.getLogger(TestTypedAdapter.class.getName());
+
public TestTypedAdapter(ITupleParserFactory parserFactory, ARecordType sourceDatatype, IHyracksTaskContext ctx,
Map<String, String> configuration, int partition) throws IOException {
- super(parserFactory, sourceDatatype, ctx, partition);
pos = new PipedOutputStream();
pis = new PipedInputStream(pos);
this.configuration = configuration;
+ this.tupleParser = parserFactory.createTupleParser(ctx);
+ this.sourceDatatype = sourceDatatype;
}
@Override
- public InputStream getInputStream(int partition) throws IOException {
- return pis;
- }
-
- @Override
- public void start(int partition, IFrameWriter frameWriter) throws Exception {
+ public void start(int partition, IFrameWriter writer) throws Exception {
generator = new DummyGenerator(configuration, pos);
ExecutorService executor = Executors.newSingleThreadExecutor();
executor.execute(generator);
- super.start(partition, frameWriter);
+ if (pis != null) {
+ tupleParser.parse(pis, writer);
+ } else {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning(
+ "Could not obtain input stream for parsing from adapter " + this + "[" + partition + "]");
+ }
+ }
}
private static class DummyGenerator implements Runnable {
@@ -135,4 +147,13 @@
return false;
}
+ @Override
+ public boolean pause() {
+ return false;
+ }
+
+ @Override
+ public boolean resume() {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
index 6b08f3a..5346bf2 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
@@ -22,9 +22,9 @@
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
import org.apache.asterix.common.parse.ITupleForwarder;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.api.IDataSourceAdapter;
import org.apache.asterix.external.parser.ADMDataParser;
import org.apache.asterix.external.util.DataflowUtils;
import org.apache.asterix.om.types.ARecordType;
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
index 71c762a..3f85ba9 100644
--- a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
@@ -24,12 +24,13 @@
import java.util.logging.Logger;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedActivity;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.api.IDataSourceAdapter;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.watch.FeedActivity;
import org.apache.asterix.lang.aql.parser.AQLParserFactory;
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.IParserFactory;
@@ -41,12 +42,9 @@
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
-import org.apache.asterix.metadata.entities.SecondaryFeed;
-import org.apache.asterix.metadata.feeds.FeedUtil;
+import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
@@ -188,14 +186,13 @@
try {
switch (feed.getFeedType()) {
case PRIMARY:
- Triple<IAdapterFactory, ARecordType, AdapterType> factoryOutput = null;
+ Triple<IAdapterFactory, ARecordType, IDataSourceAdapter.AdapterType> factoryOutput = null;
- factoryOutput = FeedUtil.getPrimaryFeedFactoryAndOutput((PrimaryFeed) feed, policyAccessor,
- mdTxnCtx);
+ factoryOutput = FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(feed, policyAccessor, mdTxnCtx);
outputType = factoryOutput.second.getTypeName();
break;
case SECONDARY:
- outputType = FeedUtil.getSecondaryFeedOutput((SecondaryFeed) feed, policyAccessor, mdTxnCtx);
+ outputType = FeedMetadataUtil.getSecondaryFeedOutput(feed, policyAccessor, mdTxnCtx);
break;
}
return outputType;
diff --git a/asterix-lang-aql/src/main/javacc/AQL.jj b/asterix-lang-aql/src/main/javacc/AQL.jj
index 12e7897..8f62f74 100644
--- a/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -43,6 +43,7 @@
import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.aql.clause.DistinctClause;
@@ -127,7 +128,6 @@
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
diff --git a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
index 49aa74b..d9d09f7 100644
--- a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
+++ b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
@@ -20,12 +20,12 @@
import java.util.Map;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
public class DatasetDecl implements Statement {
protected final Identifier name;
diff --git a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 28e5af0..a1e6363 100644
--- a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++ b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -27,6 +27,7 @@
import java.util.Map;
import java.util.Set;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -95,7 +96,6 @@
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
diff --git a/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/VariableCheckAndRewriteVisitor.java b/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/VariableCheckAndRewriteVisitor.java
index b016b62..f9cf99f 100644
--- a/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/VariableCheckAndRewriteVisitor.java
+++ b/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/VariableCheckAndRewriteVisitor.java
@@ -22,6 +22,7 @@
import java.util.Iterator;
import java.util.List;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
@@ -69,7 +70,6 @@
import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
import org.apache.asterix.lang.sqlpp.util.SqlppFormatPrintUtil;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppQueryExpressionVisitor;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.hyracks.algebricks.core.algebra.base.Counter;
public class VariableCheckAndRewriteVisitor extends AbstractSqlppQueryExpressionVisitor<Expression, Void> {
diff --git a/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 0e3fad8..547a10b 100644
--- a/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -44,6 +44,7 @@
import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
@@ -140,7 +141,6 @@
import org.apache.asterix.lang.sqlpp.optype.SetOpType;
import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
index ea65b20..a3a81de 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
@@ -35,7 +35,7 @@
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
@@ -66,9 +66,9 @@
protected final Map<FunctionSignature, Function> functions = new HashMap<FunctionSignature, Function>();
// Key is adapter dataverse. Key of value map is the adapter name
protected final Map<String, Map<String, DatasourceAdapter>> adapters = new HashMap<String, Map<String, DatasourceAdapter>>();
-
+
// Key is DataverseName, Key of the value map is the Policy name
- protected final Map<String, Map<String, FeedPolicy>> feedPolicies = new HashMap<String, Map<String, FeedPolicy>>();
+ protected final Map<String, Map<String, FeedPolicyEntity>> feedPolicies = new HashMap<String, Map<String, FeedPolicyEntity>>();
// Key is library dataverse. Key of value map is the library name
protected final Map<String, Map<String, Library>> libraries = new HashMap<String, Map<String, Library>>();
// Key is library dataverse. Key of value map is the feed name
@@ -110,18 +110,17 @@
synchronized (datatypes) {
synchronized (functions) {
synchronized (adapters) {
- synchronized (libraries) {
- synchronized (compactionPolicies) {
- dataverses.clear();
- nodeGroups.clear();
- datasets.clear();
- indexes.clear();
- datatypes.clear();
- functions.clear();
- adapters.clear();
- libraries.clear();
- compactionPolicies.clear();
- }
+ synchronized (libraries) {
+ synchronized (compactionPolicies) {
+ dataverses.clear();
+ nodeGroups.clear();
+ datasets.clear();
+ indexes.clear();
+ datatypes.clear();
+ functions.clear();
+ adapters.clear();
+ libraries.clear();
+ compactionPolicies.clear();
}
}
}
@@ -131,9 +130,9 @@
}
}
}
-
+ }
- public Object addDataverseIfNotExists(Dataverse dataverse) {
+ public Dataverse addDataverseIfNotExists(Dataverse dataverse) {
synchronized (dataverses) {
synchronized (datasets) {
synchronized (datatypes) {
@@ -149,7 +148,7 @@
}
}
- public Object addDatasetIfNotExists(Dataset dataset) {
+ public Dataset addDatasetIfNotExists(Dataset dataset) {
synchronized (datasets) {
synchronized (indexes) {
// Add the primary index associated with the dataset, if the dataset is an
@@ -175,13 +174,13 @@
}
}
- public Object addIndexIfNotExists(Index index) {
+ public Index addIndexIfNotExists(Index index) {
synchronized (indexes) {
return addIndexIfNotExistsInternal(index);
}
}
- public Object addDatatypeIfNotExists(Datatype datatype) {
+ public Datatype addDatatypeIfNotExists(Datatype datatype) {
synchronized (datatypes) {
Map<String, Datatype> m = datatypes.get(datatype.getDataverseName());
if (m == null) {
@@ -195,7 +194,7 @@
}
}
- public Object addNodeGroupIfNotExists(NodeGroup nodeGroup) {
+ public NodeGroup addNodeGroupIfNotExists(NodeGroup nodeGroup) {
synchronized (nodeGroups) {
if (!nodeGroups.containsKey(nodeGroup.getNodeGroupName())) {
return nodeGroups.put(nodeGroup.getNodeGroupName(), nodeGroup);
@@ -204,7 +203,7 @@
}
}
- public Object addCompactionPolicyIfNotExists(CompactionPolicy compactionPolicy) {
+ public CompactionPolicy addCompactionPolicyIfNotExists(CompactionPolicy compactionPolicy) {
synchronized (compactionPolicy) {
Map<String, CompactionPolicy> p = compactionPolicies.get(compactionPolicy.getDataverseName());
if (p == null) {
@@ -220,17 +219,17 @@
}
}
- public Object dropCompactionPolicy(CompactionPolicy compactionPolicy) {
+ public CompactionPolicy dropCompactionPolicy(CompactionPolicy compactionPolicy) {
synchronized (compactionPolicies) {
Map<String, CompactionPolicy> p = compactionPolicies.get(compactionPolicy.getDataverseName());
if (p != null && p.get(compactionPolicy.getPolicyName()) != null) {
- return p.remove(compactionPolicy).getPolicyName();
+ return p.remove(compactionPolicy);
}
return null;
}
}
- public Object dropDataverse(Dataverse dataverse) {
+ public Dataverse dropDataverse(Dataverse dataverse) {
synchronized (dataverses) {
synchronized (datasets) {
synchronized (indexes) {
@@ -238,26 +237,25 @@
synchronized (functions) {
synchronized (adapters) {
synchronized (libraries) {
- synchronized (feeds) {
- synchronized (compactionPolicies) {
- datasets.remove(dataverse.getDataverseName());
- indexes.remove(dataverse.getDataverseName());
- datatypes.remove(dataverse.getDataverseName());
- adapters.remove(dataverse.getDataverseName());
- compactionPolicies.remove(dataverse.getDataverseName());
- List<FunctionSignature> markedFunctionsForRemoval = new ArrayList<FunctionSignature>();
- for (FunctionSignature signature : functions.keySet()) {
- if (signature.getNamespace().equals(dataverse.getDataverseName())) {
- markedFunctionsForRemoval.add(signature);
- }
+ synchronized (feeds) {
+ synchronized (compactionPolicies) {
+ datasets.remove(dataverse.getDataverseName());
+ indexes.remove(dataverse.getDataverseName());
+ datatypes.remove(dataverse.getDataverseName());
+ adapters.remove(dataverse.getDataverseName());
+ compactionPolicies.remove(dataverse.getDataverseName());
+ List<FunctionSignature> markedFunctionsForRemoval = new ArrayList<FunctionSignature>();
+ for (FunctionSignature signature : functions.keySet()) {
+ if (signature.getNamespace().equals(dataverse.getDataverseName())) {
+ markedFunctionsForRemoval.add(signature);
}
- for (FunctionSignature signature : markedFunctionsForRemoval) {
- functions.remove(signature);
- }
- libraries.remove(dataverse.getDataverseName());
- feeds.remove(dataverse.getDataverseName());
- return dataverses.remove(dataverse.getDataverseName());
}
+ for (FunctionSignature signature : markedFunctionsForRemoval) {
+ functions.remove(signature);
+ }
+ libraries.remove(dataverse.getDataverseName());
+ feeds.remove(dataverse.getDataverseName());
+ return dataverses.remove(dataverse.getDataverseName());
}
}
}
@@ -267,9 +265,9 @@
}
}
}
-
+ }
- public Object dropDataset(Dataset dataset) {
+ public Dataset dropDataset(Dataset dataset) {
synchronized (datasets) {
synchronized (indexes) {
@@ -289,7 +287,7 @@
}
}
- public Object dropIndex(Index index) {
+ public Index dropIndex(Index index) {
synchronized (indexes) {
Map<String, Map<String, Index>> datasetMap = indexes.get(index.getDataverseName());
if (datasetMap == null) {
@@ -304,7 +302,7 @@
}
}
- public Object dropDatatype(Datatype datatype) {
+ public Datatype dropDatatype(Datatype datatype) {
synchronized (datatypes) {
Map<String, Datatype> m = datatypes.get(datatype.getDataverseName());
if (m == null) {
@@ -314,7 +312,7 @@
}
}
- public Object dropNodeGroup(NodeGroup nodeGroup) {
+ public NodeGroup dropNodeGroup(NodeGroup nodeGroup) {
synchronized (nodeGroups) {
return nodeGroups.remove(nodeGroup.getNodeGroupName());
}
@@ -405,11 +403,11 @@
*/
protected class MetadataLogicalOperation {
// Entity to be added/dropped.
- public final IMetadataEntity entity;
+ public final IMetadataEntity<?> entity;
// True for add, false for drop.
public final boolean isAdd;
- public MetadataLogicalOperation(IMetadataEntity entity, boolean isAdd) {
+ public MetadataLogicalOperation(IMetadataEntity<?> entity, boolean isAdd) {
this.entity = entity;
this.isAdd = isAdd;
}
@@ -431,7 +429,7 @@
}
}
- public Object addFunctionIfNotExists(Function function) {
+ public Function addFunctionIfNotExists(Function function) {
synchronized (functions) {
FunctionSignature signature = new FunctionSignature(function.getDataverseName(), function.getName(),
function.getArity());
@@ -443,7 +441,7 @@
}
}
- public Object dropFunction(Function function) {
+ public Function dropFunction(Function function) {
synchronized (functions) {
FunctionSignature signature = new FunctionSignature(function.getDataverseName(), function.getName(),
function.getArity());
@@ -455,11 +453,11 @@
}
}
- public Object addFeedPolicyIfNotExists(FeedPolicy feedPolicy) {
+ public Object addFeedPolicyIfNotExists(FeedPolicyEntity feedPolicy) {
synchronized (feedPolicy) {
- Map<String, FeedPolicy> p = feedPolicies.get(feedPolicy.getDataverseName());
+ Map<String, FeedPolicyEntity> p = feedPolicies.get(feedPolicy.getDataverseName());
if (p == null) {
- p = new HashMap<String, FeedPolicy>();
+ p = new HashMap<String, FeedPolicyEntity>();
p.put(feedPolicy.getPolicyName(), feedPolicy);
feedPolicies.put(feedPolicy.getDataverseName(), p);
} else {
@@ -471,9 +469,9 @@
}
}
- public Object dropFeedPolicy(FeedPolicy feedPolicy) {
+ public Object dropFeedPolicy(FeedPolicyEntity feedPolicy) {
synchronized (feedPolicies) {
- Map<String, FeedPolicy> p = feedPolicies.get(feedPolicy.getDataverseName());
+ Map<String, FeedPolicyEntity> p = feedPolicies.get(feedPolicy.getDataverseName());
if (p != null && p.get(feedPolicy.getPolicyName()) != null) {
return p.remove(feedPolicy).getPolicyName();
}
@@ -481,10 +479,10 @@
}
}
- public Object addAdapterIfNotExists(DatasourceAdapter adapter) {
+ public DatasourceAdapter addAdapterIfNotExists(DatasourceAdapter adapter) {
synchronized (adapters) {
- Map<String, DatasourceAdapter> adaptersInDataverse = adapters.get(adapter.getAdapterIdentifier()
- .getNamespace());
+ Map<String, DatasourceAdapter> adaptersInDataverse = adapters
+ .get(adapter.getAdapterIdentifier().getNamespace());
if (adaptersInDataverse == null) {
adaptersInDataverse = new HashMap<String, DatasourceAdapter>();
adapters.put(adapter.getAdapterIdentifier().getNamespace(), adaptersInDataverse);
@@ -497,10 +495,10 @@
}
}
- public Object dropAdapter(DatasourceAdapter adapter) {
+ public DatasourceAdapter dropAdapter(DatasourceAdapter adapter) {
synchronized (adapters) {
- Map<String, DatasourceAdapter> adaptersInDataverse = adapters.get(adapter.getAdapterIdentifier()
- .getNamespace());
+ Map<String, DatasourceAdapter> adaptersInDataverse = adapters
+ .get(adapter.getAdapterIdentifier().getNamespace());
if (adaptersInDataverse != null) {
return adaptersInDataverse.remove(adapter.getAdapterIdentifier().getName());
}
@@ -508,10 +506,7 @@
}
}
-
-
-
- public Object addLibraryIfNotExists(Library library) {
+ public Library addLibraryIfNotExists(Library library) {
synchronized (libraries) {
Map<String, Library> libsInDataverse = libraries.get(library.getDataverseName());
boolean needToAddd = (libsInDataverse == null || libsInDataverse.get(library.getName()) != null);
@@ -526,7 +521,7 @@
}
}
- public Object dropLibrary(Library library) {
+ public Library dropLibrary(Library library) {
synchronized (libraries) {
Map<String, Library> librariesInDataverse = libraries.get(library.getDataverseName());
if (librariesInDataverse != null) {
@@ -536,12 +531,11 @@
}
}
- public Object addFeedIfNotExists(Feed feed) {
- // TODO Auto-generated method stub
+ public Feed addFeedIfNotExists(Feed feed) {
return null;
}
- public Object dropFeed(Feed feed) {
+ public Feed dropFeed(Feed feed) {
synchronized (feeds) {
Map<String, Feed> feedsInDataverse = feeds.get(feed.getDataverseName());
if (feedsInDataverse != null) {
@@ -551,7 +545,7 @@
}
}
- private Object addIndexIfNotExistsInternal(Index index) {
+ private Index addIndexIfNotExistsInternal(Index index) {
Map<String, Map<String, Index>> datasetMap = indexes.get(index.getDataverseName());
if (datasetMap == null) {
datasetMap = new HashMap<String, Map<String, Index>>();
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index d748ef8..8f4e763 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -39,7 +39,7 @@
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.Library;
@@ -651,7 +651,8 @@
}
@Override
- public void addFeedPolicy(MetadataTransactionContext mdTxnCtx, FeedPolicy feedPolicy) throws MetadataException {
+ public void addFeedPolicy(MetadataTransactionContext mdTxnCtx, FeedPolicyEntity feedPolicy)
+ throws MetadataException {
try {
metadataNode.addFeedPolicy(mdTxnCtx.getJobId(), feedPolicy);
} catch (RemoteException e) {
@@ -786,10 +787,10 @@
}
@Override
- public FeedPolicy getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
+ public FeedPolicyEntity getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
throws MetadataException {
- FeedPolicy FeedPolicy = null;
+ FeedPolicyEntity FeedPolicy = null;
try {
FeedPolicy = metadataNode.getFeedPolicy(ctx.getJobId(), dataverse, policyName);
} catch (RemoteException e) {
@@ -844,7 +845,7 @@
public void dropFeedPolicy(MetadataTransactionContext mdTxnCtx, String dataverseName, String policyName)
throws MetadataException {
- FeedPolicy feedPolicy = null;
+ FeedPolicyEntity feedPolicy = null;
try {
feedPolicy = metadataNode.getFeedPolicy(mdTxnCtx.getJobId(), dataverseName, policyName);
metadataNode.dropFeedPolicy(mdTxnCtx.getJobId(), dataverseName, policyName);
@@ -854,9 +855,9 @@
mdTxnCtx.dropFeedPolicy(feedPolicy);
}
- public List<FeedPolicy> getDataversePolicies(MetadataTransactionContext mdTxnCtx, String dataverse)
+ public List<FeedPolicyEntity> getDataversePolicies(MetadataTransactionContext mdTxnCtx, String dataverse)
throws MetadataException {
- List<FeedPolicy> dataverseFeedPolicies;
+ List<FeedPolicyEntity> dataverseFeedPolicies;
try {
dataverseFeedPolicies = metadataNode.getDataversePolicies(mdTxnCtx.getJobId(), dataverse);
} catch (RemoteException e) {
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index ce272fa..ed586aa 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -50,7 +50,7 @@
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
@@ -364,10 +364,12 @@
dropFeed(jobId, dataverseName, feed.getFeedName());
}
- List<FeedPolicy> feedPolicies = getDataversePolicies(jobId, dataverseName);
- // Drop all feed ingestion policies in this dataverse.
- for (FeedPolicy feedPolicy : feedPolicies) {
- dropFeedPolicy(jobId, dataverseName, feedPolicy.getPolicyName());
+ List<FeedPolicyEntity> feedPolicies = getDataversePolicies(jobId, dataverseName);
+ if (feedPolicies != null && feedPolicies.size() > 0) {
+ // Drop all feed ingestion policies in this dataverse.
+ for (FeedPolicyEntity feedPolicy : feedPolicies) {
+ dropFeedPolicy(jobId, dataverseName, feedPolicy.getPolicyName());
+ }
}
// Delete the dataverse entry from the 'dataverse' dataset.
@@ -1315,7 +1317,7 @@
}
@Override
- public void addFeedPolicy(JobId jobId, FeedPolicy feedPolicy) throws MetadataException, RemoteException {
+ public void addFeedPolicy(JobId jobId, FeedPolicyEntity feedPolicy) throws MetadataException, RemoteException {
try {
// Insert into the 'FeedPolicy' dataset.
FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(true);
@@ -1332,14 +1334,14 @@
}
@Override
- public FeedPolicy getFeedPolicy(JobId jobId, String dataverse, String policyName)
+ public FeedPolicyEntity getFeedPolicy(JobId jobId, String dataverse, String policyName)
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverse, policyName);
FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(false);
- List<FeedPolicy> results = new ArrayList<FeedPolicy>();
- IValueExtractor<FeedPolicy> valueExtractor = new MetadataEntityValueExtractor<FeedPolicy>(
+ List<FeedPolicyEntity> results = new ArrayList<FeedPolicyEntity>();
+ IValueExtractor<FeedPolicyEntity> valueExtractor = new MetadataEntityValueExtractor<FeedPolicyEntity>(
tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, searchKey, valueExtractor, results);
if (!results.isEmpty()) {
@@ -1418,14 +1420,14 @@
}
@Override
- public List<FeedPolicy> getDataversePolicies(JobId jobId, String dataverse)
+ public List<FeedPolicyEntity> getDataversePolicies(JobId jobId, String dataverse)
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverse);
FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(false);
- IValueExtractor<FeedPolicy> valueExtractor = new MetadataEntityValueExtractor<FeedPolicy>(
+ IValueExtractor<FeedPolicyEntity> valueExtractor = new MetadataEntityValueExtractor<FeedPolicyEntity>(
tupleReaderWriter);
- List<FeedPolicy> results = new ArrayList<FeedPolicy>();
+ List<FeedPolicyEntity> results = new ArrayList<FeedPolicyEntity>();
searchIndex(jobId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, searchKey, valueExtractor, results);
return results;
} catch (Exception e) {
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
index b5a689d..3d07a00 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
@@ -23,6 +23,9 @@
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.api.IFeed.FeedType;
import org.apache.asterix.metadata.api.IMetadataEntity;
import org.apache.asterix.metadata.entities.CompactionPolicy;
import org.apache.asterix.metadata.entities.Dataset;
@@ -30,15 +33,11 @@
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.Feed.FeedType;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.metadata.entities.NodeGroup;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
-import org.apache.asterix.metadata.entities.SecondaryFeed;
-import org.apache.asterix.metadata.feeds.AdapterIdentifier;
/**
* Used to implement serializable transactions against the MetadataCache.
@@ -223,7 +222,7 @@
return opLog;
}
- public void addFeedPolicy(FeedPolicy feedPolicy) {
+ public void addFeedPolicy(FeedPolicyEntity feedPolicy) {
droppedCache.dropFeedPolicy(feedPolicy);
logAndApply(new MetadataLogicalOperation(feedPolicy, true));
@@ -235,16 +234,10 @@
}
- public void dropFeed(String dataverseName, String feedName, FeedType feedType) {
+ public void dropFeed(String dataverseName, String feedName, IFeed.FeedType feedType) {
Feed feed = null;
- switch (feedType) {
- case PRIMARY:
- feed = new PrimaryFeed(dataverseName, feedName, null, null, null);
- break;
- case SECONDARY:
- feed = new SecondaryFeed(dataverseName, feedName, null, null);
- break;
- }
+ feed = new Feed(dataverseName, feedName, null, feedType, (feedType == FeedType.PRIMARY) ? feedName : null, null,
+ null);
droppedCache.addFeedIfNotExists(feed);
logAndApply(new MetadataLogicalOperation(feed, false));
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java
index bc19f72..2fc846b 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java
@@ -23,13 +23,13 @@
import org.apache.asterix.metadata.MetadataCache;
-public interface IMetadataEntity extends Serializable {
+public interface IMetadataEntity<T> extends Serializable {
public static final int PENDING_NO_OP = 0;
public static final int PENDING_ADD_OP = 1;
public static final int PENDING_DROP_OP = 2;
- Object addToCache(MetadataCache cache);
+ public T addToCache(MetadataCache cache);
- Object dropFromCache(MetadataCache cache);
+ public T dropFromCache(MetadataCache cache);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index 9d85fb5..242bbe3 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -33,7 +33,7 @@
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.Library;
@@ -56,7 +56,6 @@
/**
* Initializes the metadata manager, e.g., finds the remote metadata node.
- *
* @throws RemoteException
* If an error occurred while contacting the proxy for finding
* the metadata node.
@@ -65,7 +64,6 @@
/**
* Begins a transaction on the metadata node.
- *
* @return A globally unique transaction id.
* @throws ACIDException
* @throws RemoteException
@@ -74,7 +72,6 @@
/**
* Commits a remote transaction on the metadata node.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @throws ACIDException
@@ -84,7 +81,6 @@
/**
* Aborts a remote transaction running on the metadata node.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @throws ACIDException
@@ -95,7 +91,6 @@
/**
* Locks the metadata in given mode. The lock acquisition is delegated to
* the metadata node. This method blocks until the lock can be acquired.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param lockMode
@@ -107,7 +102,6 @@
/**
* Releases all locks on the metadata held by the given transaction id.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @throws ACIDException
@@ -117,7 +111,6 @@
/**
* Inserts a new dataverse into the metadata.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverse
@@ -129,7 +122,6 @@
/**
* Retrieves all dataverses
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @return A list of dataverse instances.
@@ -139,7 +131,6 @@
/**
* Retrieves a dataverse with given name.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -152,7 +143,6 @@
/**
* Retrieves all datasets belonging to the given dataverse.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -167,7 +157,6 @@
/**
* Deletes the dataverse with given name, and all it's associated datasets,
* indexes, and types.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @return A list of dataset instances.
@@ -178,7 +167,6 @@
/**
* Inserts a new dataset into the metadata.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataset
@@ -190,7 +178,6 @@
/**
* Retrieves a dataset within a given dataverse.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -206,7 +193,6 @@
/**
* Retrieves all indexes of a dataset.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -222,7 +208,6 @@
/**
* Deletes the dataset with given name, and all it's associated indexes.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -238,7 +223,6 @@
/**
* Inserts an index into the metadata. The index itself knows its name, and
* which dataset it belongs to.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param index
@@ -250,7 +234,6 @@
/**
* Retrieves the index with given name, in given dataverse and dataset.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -267,7 +250,6 @@
/**
* Deletes the index with given name, in given dataverse and dataset.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -283,7 +265,6 @@
/**
* Inserts a datatype.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param datatype
@@ -295,7 +276,6 @@
/**
* Retrieves the datatype with given name in given dataverse.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -311,7 +291,6 @@
/**
* Deletes the given datatype in given dataverse.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -327,7 +306,6 @@
/**
* Inserts a node group.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param nodeGroup
@@ -339,7 +317,6 @@
/**
* Retrieves a node group.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param nodeGroupName
@@ -351,7 +328,6 @@
/**
* Deletes a node group.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param nodeGroupName
@@ -364,7 +340,6 @@
/**
* Inserts a node (machine).
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param node
@@ -490,13 +465,12 @@
*/
public void dropFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws MetadataException;
-
- /**
+ /**
* @param ctx
* @param policy
* @throws MetadataException
*/
- public void addFeedPolicy(MetadataTransactionContext ctx, FeedPolicy policy) throws MetadataException;
+ public void addFeedPolicy(MetadataTransactionContext ctx, FeedPolicyEntity policy) throws MetadataException;
/**
* @param ctx
@@ -505,10 +479,9 @@
* @return
* @throws MetadataException
*/
- public FeedPolicy getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
+ public FeedPolicyEntity getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
throws MetadataException;
-
public void initializeDatasetIdFactory(MetadataTransactionContext ctx) throws MetadataException;
public int getMostRecentDatasetId() throws MetadataException;
@@ -524,7 +497,6 @@
/**
* Removes a library , acquiring local locks on behalf of the given
* transaction id.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -540,7 +512,6 @@
/**
* Adds a library, acquiring local locks on behalf of the given
* transaction id.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param library
@@ -567,7 +538,6 @@
/**
* Retireve libraries installed in a given dataverse.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -624,7 +594,6 @@
/**
* Get en external file
- *
* @param mdTxnCtx
* @param dataverseName
* @param datasetName
@@ -637,7 +606,6 @@
/**
* update an existing dataset in metadata.
- *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataset
@@ -649,7 +617,6 @@
/**
* Clean up temporary datasets that have not been active for a long time.
- *
* @throws MetadataException
*/
public void cleanupTempDatasets() throws MetadataException;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index 59a8f76..0d8818d 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -35,7 +35,7 @@
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.Library;
@@ -557,7 +557,7 @@
* @throws MetadataException
* @throws RemoteException
*/
- public void addFeedPolicy(JobId jobId, FeedPolicy feedPolicy) throws MetadataException, RemoteException;
+ public void addFeedPolicy(JobId jobId, FeedPolicyEntity feedPolicy) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -567,7 +567,7 @@
* @throws MetadataException
* @throws RemoteException
*/
- public FeedPolicy getFeedPolicy(JobId jobId, String dataverse, String policy) throws MetadataException,
+ public FeedPolicyEntity getFeedPolicy(JobId jobId, String dataverse, String policy) throws MetadataException,
RemoteException;
@@ -658,7 +658,7 @@
* @throws MetadataException
* @throws RemoteException
*/
- public List<FeedPolicy> getDataversePolicies(JobId jobId, String dataverse) throws MetadataException,
+ public List<FeedPolicyEntity> getDataversePolicies(JobId jobId, String dataverse) throws MetadataException,
RemoteException;
/**
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index ab0b79d..aa7f7d5 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.metadata.bootstrap;
import java.io.File;
@@ -37,14 +36,16 @@
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.config.IAsterixPropertiesProvider;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.adapter.factory.GenericAdapterFactory;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.api.IDataSourceAdapter;
+import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.external.runtime.GenericSocketFeedAdapterFactory;
-import org.apache.asterix.external.runtime.SocketClientAdapterFactory;
import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
@@ -57,16 +58,14 @@
import org.apache.asterix.metadata.entities.DatasourceAdapter;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
import org.apache.asterix.metadata.entities.Node;
import org.apache.asterix.metadata.entities.NodeGroup;
-import org.apache.asterix.metadata.feeds.AdapterIdentifier;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
-import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.util.AsterixClusterProperties;
@@ -105,7 +104,7 @@
*/
public class MetadataBootstrap {
private static final Logger LOGGER = Logger.getLogger(MetadataBootstrap.class.getName());
- public static final boolean IS_DEBUG_MODE = false;//true
+ public static final boolean IS_DEBUG_MODE = false;// true
private static IAsterixAppRuntimeContext runtimeContext;
@@ -193,7 +192,7 @@
}
}
- //#. initialize datasetIdFactory
+ // #. initialize datasetIdFactory
MetadataManager.INSTANCE.initializeDatasetIdFactory(mdTxnCtx);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
@@ -204,8 +203,8 @@
MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
} catch (Exception e2) {
e.addSuppressed(e2);
- //TODO
- //change the exception type to AbortFailureException
+ // TODO
+ // change the exception type to AbortFailureException
throw new MetadataException(e);
}
throw e;
@@ -278,8 +277,8 @@
// Map.Entry<String, String[]> me = (Map.Entry<String,
// String[]>)im.next();
MetadataManager.INSTANCE.addNode(mdTxnCtx, new Node(iter.next(), 0, 0/*
- * , me . getValue ( )
- */));
+ * , me . getValue ( )
+ */));
}
}
@@ -298,8 +297,7 @@
}
private static void insertInitialAdapters(MetadataTransactionContext mdTxnCtx) throws Exception {
- String[] builtInAdapterClassNames = new String[] { GenericAdapterFactory.class.getName(),
- GenericSocketFeedAdapterFactory.class.getName(), SocketClientAdapterFactory.class.getName() };
+ String[] builtInAdapterClassNames = new String[] { GenericAdapterFactory.class.getName() };
DatasourceAdapter adapter;
for (String adapterClassName : builtInAdapterClassNames) {
adapter = getAdapter(adapterClassName);
@@ -311,7 +309,7 @@
}
private static void insertInitialFeedPolicies(MetadataTransactionContext mdTxnCtx) throws Exception {
- for (FeedPolicy feedPolicy : BuiltinFeedPolicies.policies) {
+ for (FeedPolicyEntity feedPolicy : BuiltinFeedPolicies.policies) {
MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, feedPolicy);
}
if (LOGGER.isLoggable(Level.INFO)) {
@@ -333,7 +331,7 @@
private static DatasourceAdapter getAdapter(String adapterFactoryClassName) throws Exception {
String adapterName = ((IAdapterFactory) (Class.forName(adapterFactoryClassName).newInstance())).getAlias();
return new DatasourceAdapter(new AdapterIdentifier(MetadataConstants.METADATA_DATAVERSE_NAME, adapterName),
- adapterFactoryClassName, DatasourceAdapter.AdapterType.INTERNAL);
+ adapterFactoryClassName, IDataSourceAdapter.AdapterType.INTERNAL);
}
private static CompactionPolicy getCompactionPolicyEntity(String compactionPolicyClassName) throws Exception {
@@ -346,7 +344,7 @@
throws Exception {
ClusterPartition metadataPartition = propertiesProvider.getMetadataProperties().getMetadataPartition();
int metadataDeviceId = metadataPartition.getIODeviceNum();
- String metadataPartitionPath = SplitsAndConstraintsUtil.prepareStoragePartitionPath(
+ String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath(
AsterixClusterProperties.INSTANCE.getStorageDirectoryName(), metadataPartition.getPartitionId());
String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath();
FileReference file = ioManager.getAbsoluteFileRef(metadataDeviceId, resourceName);
@@ -413,8 +411,8 @@
}
public static void startDDLRecovery() throws MetadataException {
- //#. clean up any record which has pendingAdd/DelOp flag
- // as traversing all records from DATAVERSE_DATASET to DATASET_DATASET, and then to INDEX_DATASET.
+ // #. clean up any record which has pendingAdd/DelOp flag
+ // as traversing all records from DATAVERSE_DATASET to DATASET_DATASET, and then to INDEX_DATASET.
String dataverseName = null;
String datasetName = null;
String indexName = null;
@@ -433,7 +431,7 @@
for (Dataverse dataverse : dataverses) {
dataverseName = dataverse.getDataverseName();
if (dataverse.getPendingOp() != IMetadataEntity.PENDING_NO_OP) {
- //drop pending dataverse
+ // drop pending dataverse
MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverseName);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Dropped a pending dataverse: " + dataverseName);
@@ -443,7 +441,7 @@
for (Dataset dataset : datasets) {
datasetName = dataset.getDatasetName();
if (dataset.getPendingOp() != IMetadataEntity.PENDING_NO_OP) {
- //drop pending dataset
+ // drop pending dataset
MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Dropped a pending dataset: " + dataverseName + "." + datasetName);
@@ -454,7 +452,7 @@
for (Index index : indexes) {
indexName = index.getIndexName();
if (index.getPendingOp() != IMetadataEntity.PENDING_NO_OP) {
- //drop pending index
+ // drop pending index
MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Dropped a pending index: " + dataverseName + "." + datasetName
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
index 9ef9f84..2d5e663 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
@@ -24,6 +24,7 @@
import java.util.Arrays;
import java.util.List;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixRuntimeException;
import org.apache.asterix.common.transactions.DatasetId;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java
index 129bc0c..91d25b3 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.bootstrap;
+import org.apache.asterix.common.config.MetadataConstants;
+
public enum MetadataIndexImmutableProperties {
METADATA(MetadataConstants.METADATA_DATAVERSE_NAME, 0, 0),
DATAVERSE("Dataverse", 1, 1),
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index ce22177..cff5c6b 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.metadata.declared;
import java.io.File;
@@ -34,6 +33,7 @@
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.ITransactionSubsystemProvider;
import org.apache.asterix.common.context.TransactionSubsystemProvider;
@@ -41,27 +41,32 @@
import org.apache.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.dataflow.IAsterixApplicationContextInfo;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedActivity;
-import org.apache.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConstants;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.api.ICentralFeedManager;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
import org.apache.asterix.external.adapter.factory.LookupAdapterFactory;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.api.IDataSourceAdapter;
+import org.apache.asterix.external.feed.api.ICentralFeedManager;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.watch.FeedActivity;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.external.operators.ExternalBTreeSearchOperatorDescriptor;
+import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalLookupOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalRTreeSearchOperatorDescriptor;
+import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
+import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
import org.apache.asterix.external.provider.AdapterFactoryProvider;
+import org.apache.asterix.external.util.FeedConstants;
import org.apache.asterix.formats.base.IDataFormat;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlLinearizeComparatorFactoryProvider;
@@ -69,25 +74,19 @@
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetCardinalityHint;
import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
-import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
-import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.FeedCollectOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.FeedIntakeOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.FeedUtil;
+import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
@@ -388,7 +387,7 @@
.getSerializerDeserializer(feedOutputType);
RecordDescriptor feedDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde });
- FeedPolicy feedPolicy = (FeedPolicy) ((AqlDataSource) dataSource).getProperties()
+ FeedPolicyEntity feedPolicy = (FeedPolicyEntity) ((AqlDataSource) dataSource).getProperties()
.get(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY);
if (feedPolicy == null) {
throw new AlgebricksException("Feed not configured with a policy");
@@ -560,7 +559,7 @@
}
}
// TODO Check this call, result of merge from master!
- // ((IGenericAdapterFactory) adapterFactory).setFiles(files);
+ // ((IGenericAdapterFactory) adapterFactory).setFiles(files);
}
return adapterFactory;
@@ -594,9 +593,9 @@
}
public Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> buildFeedIntakeRuntime(
- JobSpecification jobSpec, PrimaryFeed primaryFeed, FeedPolicyAccessor policyAccessor) throws Exception {
- Triple<IAdapterFactory, ARecordType, AdapterType> factoryOutput = null;
- factoryOutput = FeedUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx);
+ JobSpecification jobSpec, Feed primaryFeed, FeedPolicyAccessor policyAccessor) throws Exception {
+ Triple<IAdapterFactory, ARecordType, IDataSourceAdapter.AdapterType> factoryOutput = null;
+ factoryOutput = FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx);
IAdapterFactory adapterFactory = factoryOutput.first;
FeedIntakeOperatorDescriptor feedIngestor = null;
switch (factoryOutput.third) {
@@ -605,7 +604,7 @@
factoryOutput.second, policyAccessor);
break;
case EXTERNAL:
- String libraryName = primaryFeed.getAdaptorName().trim()
+ String libraryName = primaryFeed.getAdapterName().trim()
.split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[0];
feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, primaryFeed, libraryName,
adapterFactory.getClass().getName(), factoryOutput.second, policyAccessor);
@@ -2084,7 +2083,7 @@
public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataset(
String dataverseName, String datasetName, String targetIdxName, boolean temp) throws AlgebricksException {
FileSplit[] splits = splitsForDataset(mdTxnCtx, dataverseName, datasetName, targetIdxName, temp);
- return SplitsAndConstraintsUtil.splitProviderAndPartitionConstraints(splits);
+ return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataverse(
@@ -2140,7 +2139,7 @@
}
}
- public FeedPolicy findFeedPolicy(String dataverse, String policyName) throws AlgebricksException {
+ public FeedPolicyEntity findFeedPolicy(String dataverse, String policyName) throws AlgebricksException {
try {
return MetadataManager.INSTANCE.getFeedPolicy(mdTxnCtx, dataverse, policyName);
} catch (MetadataException e) {
@@ -2193,7 +2192,8 @@
itemType = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
dataset.getDataverseName(), dataset.getItemTypeName()).getDatatype();
- // Create the adapter factory <- right now there is only one. if there are more in the future, we can create a map->
+ // Create the adapter factory <- right now there is only one. if there are more in the future, we can create
+ // a map->
ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
LookupAdapterFactory<?> adapterFactory = AdapterFactoryProvider.getAdapterFactory(
datasetDetails.getProperties(), (ARecordType) itemType, ridIndexes, retainInput, retainNull,
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index 2e603f1..6f540df 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -18,12 +18,12 @@
*/
package org.apache.asterix.metadata.declared;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.management.FeedId;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.Feed.FeedType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -33,14 +33,14 @@
private Feed feed;
private final FeedId sourceFeedId;
- private final FeedType sourceFeedType;
+ private final IFeed.FeedType sourceFeedType;
private final ConnectionLocation location;
private final String targetDataset;
private final String[] locations;
private final int computeCardinality;
public FeedDataSource(AqlSourceId id, String targetDataset, IAType itemType, AqlDataSourceType dataSourceType,
- FeedId sourceFeedId, FeedType sourceFeedType, ConnectionLocation location, String[] locations)
+ FeedId sourceFeedId, IFeed.FeedType sourceFeedType, ConnectionLocation location, String[] locations)
throws AlgebricksException {
super(id, itemType, dataSourceType);
this.targetDataset = targetDataset;
@@ -75,6 +75,11 @@
return feed;
}
+ @Override
+ public IAType[] getSchemaTypes() {
+ return schemaTypes;
+ }
+
public String getTargetDataset() {
return targetDataset;
}
@@ -108,7 +113,7 @@
domain = domainForExternalData;
}
- public FeedType getSourceFeedType() {
+ public IFeed.FeedType getSourceFeedType() {
return sourceFeedType;
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/CompactionPolicy.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/CompactionPolicy.java
index 39654e9..d71663a 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/CompactionPolicy.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/CompactionPolicy.java
@@ -25,7 +25,7 @@
/**
* Metadata describing a compaction policy record.
*/
-public class CompactionPolicy implements IMetadataEntity {
+public class CompactionPolicy implements IMetadataEntity<CompactionPolicy> {
private static final long serialVersionUID = 1L;
@@ -71,12 +71,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public CompactionPolicy addToCache(MetadataCache cache) {
return cache.addCompactionPolicyIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public CompactionPolicy dropFromCache(MetadataCache cache) {
return cache.dropCompactionPolicy(this);
}
}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index ae4c742..32456ae 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -29,7 +29,7 @@
/**
* Metadata describing a dataset.
*/
-public class Dataset implements IMetadataEntity {
+public class Dataset implements IMetadataEntity<Dataset> {
private static final long serialVersionUID = 1L;
@@ -122,12 +122,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public Dataset addToCache(MetadataCache cache) {
return cache.addDatasetIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public Dataset dropFromCache(MetadataCache cache) {
return cache.dropDataset(this);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/DatasourceAdapter.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/DatasourceAdapter.java
index ac98fb0b..5a85327 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/DatasourceAdapter.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/DatasourceAdapter.java
@@ -18,19 +18,15 @@
*/
package org.apache.asterix.metadata.entities;
+import org.apache.asterix.external.api.IDataSourceAdapter.AdapterType;
+import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
-import org.apache.asterix.metadata.feeds.AdapterIdentifier;
-public class DatasourceAdapter implements IMetadataEntity {
+public class DatasourceAdapter implements IMetadataEntity<DatasourceAdapter> {
private static final long serialVersionUID = 1L;
- public enum AdapterType {
- INTERNAL,
- EXTERNAL
- }
-
private final AdapterIdentifier adapterIdentifier;
private final String classname;
private final AdapterType type;
@@ -42,12 +38,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public DatasourceAdapter addToCache(MetadataCache cache) {
return cache.addAdapterIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public DatasourceAdapter dropFromCache(MetadataCache cache) {
return cache.dropAdapter(this);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Datatype.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Datatype.java
index 73015df..0580756 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Datatype.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Datatype.java
@@ -26,7 +26,7 @@
/**
* Metadata describing a datatype.
*/
-public class Datatype implements IMetadataEntity {
+public class Datatype implements IMetadataEntity<Datatype> {
private static final long serialVersionUID = 1L;
@@ -60,12 +60,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public Datatype addToCache(MetadataCache cache) {
return cache.addDatatypeIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public Datatype dropFromCache(MetadataCache cache) {
return cache.dropDatatype(this);
}
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java
index ef315f4..467e6f5 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java
@@ -25,7 +25,7 @@
/**
* Metadata describing a dataverse.
*/
-public class Dataverse implements IMetadataEntity {
+public class Dataverse implements IMetadataEntity<Dataverse> {
private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster..
@@ -52,12 +52,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public Dataverse addToCache(MetadataCache cache) {
return cache.addDataverseIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public Dataverse dropFromCache(MetadataCache cache) {
return cache.dropDataverse(this);
}
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Feed.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Feed.java
index 5419dfa..58aa622 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Feed.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Feed.java
@@ -19,66 +19,68 @@
package org.apache.asterix.metadata.entities;
-import org.apache.asterix.common.feeds.FeedId;
+import java.util.Map;
+
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.management.FeedId;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
/**
* Feed POJO
*/
-public class Feed implements IMetadataEntity {
-
+public class Feed implements IMetadataEntity<Feed>, IFeed {
private static final long serialVersionUID = 1L;
/** A unique identifier for the feed */
- protected final FeedId feedId;
-
+ private FeedId feedId;
/** The function that is to be applied on each incoming feed tuple **/
- protected final FunctionSignature appliedFunction;
-
+ private FunctionSignature appliedFunction;
/** The type {@code FeedType} associated with the feed. **/
- protected final FeedType feedType;
-
+ private IFeed.FeedType feedType;
/** A string representation of the instance **/
- protected final String displayName;
+ private String displayName;
+ /** A string representation of the adapter name **/
+ private String adapterName;
+ /** Adapter configuration */
+ private Map<String, String> adapterConfiguration;
+ /** Source primary feed */
+ private String sourceFeedName;
- public enum FeedType {
- /**
- * A feed that derives its data from an external source.
- */
- PRIMARY,
-
- /**
- * A feed that derives its data from another primary or secondary feed.
- */
- SECONDARY
- }
-
- public Feed(String dataverseName, String datasetName, FunctionSignature appliedFunction, FeedType feedType) {
- this.feedId = new FeedId(dataverseName, datasetName);
+ public Feed(String dataverseName, String feedName, FunctionSignature appliedFunction, IFeed.FeedType feedType,
+ String sourceFeedName, String adapterName, Map<String, String> configuration) {
+ this.feedId = new FeedId(dataverseName, feedName);
this.appliedFunction = appliedFunction;
this.feedType = feedType;
this.displayName = feedType + "(" + feedId + ")";
+ this.adapterName = adapterName;
+ this.adapterConfiguration = configuration;
+ this.sourceFeedName = sourceFeedName;
}
+ @Override
public FeedId getFeedId() {
return feedId;
}
+ @Override
public String getDataverseName() {
return feedId.getDataverse();
}
+ @Override
public String getFeedName() {
return feedId.getFeedName();
}
+ @Override
public FunctionSignature getAppliedFunction() {
return appliedFunction;
}
- public FeedType getFeedType() {
+ @Override
+ public IFeed.FeedType getFeedType() {
return feedType;
}
@@ -105,12 +107,26 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public Feed addToCache(MetadataCache cache) {
return cache.addFeedIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public Feed dropFromCache(MetadataCache cache) {
return cache.dropFeed(this);
}
+
+ @Override
+ public String getAdapterName() {
+ return adapterName;
+ }
+
+ @Override
+ public Map<String, String> getAdapterConfiguration() {
+ return adapterConfiguration;
+ }
+
+ public String getSourceFeedName() {
+ return sourceFeedName;
+ }
}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedPolicyEntity.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedPolicyEntity.java
new file mode 100644
index 0000000..76f704a
--- /dev/null
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedPolicyEntity.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.metadata.entities;
+
+import java.util.Map;
+
+import org.apache.asterix.external.feed.policy.FeedPolicy;
+import org.apache.asterix.metadata.MetadataCache;
+import org.apache.asterix.metadata.api.IMetadataEntity;
+
+/**
+ * Metadata describing a feed activity record.
+ */
+public class FeedPolicyEntity extends FeedPolicy implements IMetadataEntity<FeedPolicyEntity> {
+
+ public FeedPolicyEntity(String dataverseName, String policyName, String description,
+ Map<String, String> properties) {
+ super(dataverseName, policyName, description, properties);
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FeedPolicyEntity addToCache(MetadataCache cache) {
+ return null;
+ }
+
+ @Override
+ public FeedPolicyEntity dropFromCache(MetadataCache cache) {
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
index cf7a95c..7ff423c 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
@@ -23,7 +23,7 @@
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
-public class Function implements IMetadataEntity {
+public class Function implements IMetadataEntity<Function> {
private static final long serialVersionUID = 1L;
public static final String LANGUAGE_AQL = "AQL";
public static final String LANGUAGE_JAVA = "JAVA";
@@ -85,12 +85,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public Function addToCache(MetadataCache cache) {
return cache.addFunctionIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public Function dropFromCache(MetadataCache cache) {
return cache.dropFunction(this);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
index 3c0feb9..6d047a2 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
@@ -35,7 +35,7 @@
/**
* Metadata describing an index.
*/
-public class Index implements IMetadataEntity, Comparable<Index> {
+public class Index implements IMetadataEntity<Index>, Comparable<Index> {
private static final long serialVersionUID = 1L;
@@ -202,12 +202,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public Index addToCache(MetadataCache cache) {
return cache.addIndexIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public Index dropFromCache(MetadataCache cache) {
return cache.dropIndex(this);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Library.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Library.java
index 277fdc1..8ffc266 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Library.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Library.java
@@ -21,7 +21,7 @@
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
-public class Library implements IMetadataEntity {
+public class Library implements IMetadataEntity<Library> {
private static final long serialVersionUID = 1L;
@@ -42,12 +42,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public Library addToCache(MetadataCache cache) {
return cache.addLibraryIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public Library dropFromCache(MetadataCache cache) {
return cache.dropLibrary(this);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 8ef04e2..e5088aa 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -27,7 +27,7 @@
/**
* Metadata describing a named group of compute nodes.
*/
-public class NodeGroup implements IMetadataEntity {
+public class NodeGroup implements IMetadataEntity<NodeGroup> {
private static final long serialVersionUID = 1L;
@@ -49,12 +49,12 @@
}
@Override
- public Object addToCache(MetadataCache cache) {
+ public NodeGroup addToCache(MetadataCache cache) {
return cache.addNodeGroupIfNotExists(this);
}
@Override
- public Object dropFromCache(MetadataCache cache) {
+ public NodeGroup dropFromCache(MetadataCache cache) {
return cache.dropNodeGroup(this);
}
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/PrimaryFeed.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/PrimaryFeed.java
deleted file mode 100644
index c70a210..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/PrimaryFeed.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.metadata.entities;
-
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.asterix.common.functions.FunctionSignature;
-import org.apache.asterix.metadata.api.IMetadataEntity;
-
-/**
- * A primary feed is one that derives its data from an external source via an adaptor.
- * This class is a holder object for the metadata associated with a primary feed.
- */
-public class PrimaryFeed extends Feed implements IMetadataEntity {
-
- private static final long serialVersionUID = 1L;
-
- private final String adaptorName;
- private final Map<String, String> adaptorConfiguration;
-
- public PrimaryFeed(String dataverseName, String datasetName, String adaptorName,
- Map<String, String> adaptorConfiguration, FunctionSignature appliedFunction) {
- super(dataverseName, datasetName, appliedFunction, FeedType.PRIMARY);
- this.adaptorName = adaptorName;
- this.adaptorConfiguration = adaptorConfiguration;
- }
-
- public String getAdaptorName() {
- return adaptorName;
- }
-
- public Map<String, String> getAdaptorConfiguration() {
- return adaptorConfiguration;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
- if (!super.equals(other) || !(other instanceof PrimaryFeed)) {
- return false;
- }
-
- PrimaryFeed otherFeed = (PrimaryFeed) other;
- if (!otherFeed.getAdaptorName().equals(adaptorName)) {
- return false;
- }
-
- for (Entry<String, String> entry : adaptorConfiguration.entrySet()) {
- if (!(entry.getValue().equals(otherFeed.getAdaptorConfiguration().get(entry.getKey())))) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public String toString() {
- return "PrimaryFeed (" + adaptorName + ")";
- }
-}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/SecondaryFeed.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/SecondaryFeed.java
deleted file mode 100644
index caa633a..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/SecondaryFeed.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.metadata.entities;
-
-import org.apache.asterix.common.functions.FunctionSignature;
-import org.apache.asterix.metadata.api.IMetadataEntity;
-
-/**
- * A secondary feed is one that derives its data from another (primary/secondary) feed.
- * This class is a holder object for the metadata associated with a secondary feed.
- */
-public class SecondaryFeed extends Feed implements IMetadataEntity {
-
- private static final long serialVersionUID = 1L;
-
- private final String sourceFeedName;
-
- public SecondaryFeed(String dataverseName, String feedName, String sourceFeedName, FunctionSignature appliedFunction) {
- super(dataverseName, feedName, appliedFunction, FeedType.SECONDARY);
- this.sourceFeedName = sourceFeedName;
- }
-
- public String getSourceFeedName() {
- return sourceFeedName;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
- if (!super.equals(other) || !(other instanceof SecondaryFeed)) {
- return false;
- }
-
- SecondaryFeed otherFeed = (SecondaryFeed) other;
- if (!otherFeed.getSourceFeedName().equals(sourceFeedName)) {
- return false;
- }
- return true;
- }
-
- @Override
- public String toString() {
- return "SecondaryFeed (" + feedId + ")" + "<--" + "(" + sourceFeedName + ")";
- }
-}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
index c37230d..b0d5f76 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
@@ -26,13 +26,14 @@
import java.util.Calendar;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.IDataSourceAdapter;
+import org.apache.asterix.external.api.IDataSourceAdapter.AdapterType;
+import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
-import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
-import org.apache.asterix.metadata.feeds.AdapterIdentifier;
import org.apache.asterix.om.base.ARecord;
import org.apache.asterix.om.base.AString;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -76,7 +77,7 @@
.getValueByPos(MetadataRecordTypes.DATASOURCE_ADAPTER_ARECORD_NAME_FIELD_INDEX)).getStringValue();
String classname = ((AString) adapterRecord
.getValueByPos(MetadataRecordTypes.DATASOURCE_ADAPTER_ARECORD_CLASSNAME_FIELD_INDEX)).getStringValue();
- AdapterType adapterType = AdapterType.valueOf(((AString) adapterRecord
+ IDataSourceAdapter.AdapterType adapterType = IDataSourceAdapter.AdapterType.valueOf(((AString) adapterRecord
.getValueByPos(MetadataRecordTypes.DATASOURCE_ADAPTER_ARECORD_TYPE_FIELD_INDEX)).getStringValue());
return new DatasourceAdapter(new AdapterIdentifier(dataverseName, adapterName), classname, adapterType);
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
index e09928b..00e3e63 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
@@ -35,7 +35,7 @@
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.base.ARecord;
@@ -52,7 +52,7 @@
/**
* Translates a Dataset metadata entity to an ITupleReference and vice versa.
*/
-public class FeedPolicyTupleTranslator extends AbstractTupleTranslator<FeedPolicy> {
+public class FeedPolicyTupleTranslator extends AbstractTupleTranslator<FeedPolicyEntity> {
// Field indexes of serialized FeedPolicy in a tuple.
// Key field.
public static final int FEED_POLICY_DATAVERSE_NAME_FIELD_INDEX = 0;
@@ -74,7 +74,7 @@
}
@Override
- public FeedPolicy getMetadataEntityFromTuple(ITupleReference frameTuple) throws IOException {
+ public FeedPolicyEntity getMetadataEntityFromTuple(ITupleReference frameTuple) throws IOException {
byte[] serRecord = frameTuple.getFieldData(FEED_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
int recordStartOffset = frameTuple.getFieldStart(FEED_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
int recordLength = frameTuple.getFieldLength(FEED_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
@@ -84,8 +84,8 @@
return createFeedPolicyFromARecord(feedPolicyRecord);
}
- private FeedPolicy createFeedPolicyFromARecord(ARecord feedPolicyRecord) {
- FeedPolicy feedPolicy = null;
+ private FeedPolicyEntity createFeedPolicyFromARecord(ARecord feedPolicyRecord) {
+ FeedPolicyEntity feedPolicy = null;
String dataverseName = ((AString) feedPolicyRecord
.getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX)).getStringValue();
String policyName = ((AString) feedPolicyRecord
@@ -106,12 +106,12 @@
policyParamters.put(key, value);
}
- feedPolicy = new FeedPolicy(dataverseName, policyName, description, policyParamters);
+ feedPolicy = new FeedPolicyEntity(dataverseName, policyName, description, policyParamters);
return feedPolicy;
}
@Override
- public ITupleReference getTupleFromMetadataEntity(FeedPolicy feedPolicy) throws IOException, MetadataException {
+ public ITupleReference getTupleFromMetadataEntity(FeedPolicyEntity feedPolicy) throws IOException, MetadataException {
// write the key in the first three fields of the tuple
ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java
index 09e193a..dc9fb50 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java
@@ -33,14 +33,13 @@
import org.apache.asterix.builders.RecordBuilder;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.api.IFeed.FeedType;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.Feed.FeedType;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
-import org.apache.asterix.metadata.entities.SecondaryFeed;
import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.base.ANull;
import org.apache.asterix.om.base.ARecord;
@@ -82,7 +81,7 @@
int recordLength = frameTuple.getFieldLength(FEED_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord feedRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord feedRecord = recordSerDes.deserialize(in);
return createFeedFromARecord(feedRecord);
}
@@ -103,18 +102,18 @@
String feedType = ((AString) feedRecord.getValueByPos(MetadataRecordTypes.FEED_ARECORD_FEED_TYPE_FIELD_INDEX))
.getStringValue();
- FeedType feedTypeEnum = FeedType.valueOf(feedType.toUpperCase());
+ IFeed.FeedType feedTypeEnum = IFeed.FeedType.valueOf(feedType.toUpperCase());
switch (feedTypeEnum) {
case PRIMARY: {
ARecord feedTypeDetailsRecord = (ARecord) feedRecord
.getValueByPos(MetadataRecordTypes.FEED_ARECORD_PRIMARY_TYPE_DETAILS_FIELD_INDEX);
String adapterName = ((AString) feedTypeDetailsRecord
.getValueByPos(MetadataRecordTypes.FEED_ARECORD_PRIMARY_FIELD_DETAILS_ADAPTOR_NAME_FIELD_INDEX))
- .getStringValue();
+ .getStringValue();
- IACursor cursor = ((AUnorderedList) feedTypeDetailsRecord
- .getValueByPos(MetadataRecordTypes.FEED_ARECORD_PRIMARY_FIELD_DETAILS_ADAPTOR_CONFIGURATION_FIELD_INDEX))
- .getCursor();
+ IACursor cursor = ((AUnorderedList) feedTypeDetailsRecord.getValueByPos(
+ MetadataRecordTypes.FEED_ARECORD_PRIMARY_FIELD_DETAILS_ADAPTOR_CONFIGURATION_FIELD_INDEX))
+ .getCursor();
String key;
String value;
Map<String, String> adaptorConfiguration = new HashMap<String, String>();
@@ -126,7 +125,8 @@
.getStringValue();
adaptorConfiguration.put(key, value);
}
- feed = new PrimaryFeed(dataverseName, feedName, adapterName, adaptorConfiguration, signature);
+ feed = new Feed(dataverseName, feedName, signature, FeedType.PRIMARY, feedName, adapterName,
+ adaptorConfiguration);
}
break;
@@ -136,9 +136,9 @@
String sourceFeedName = ((AString) feedTypeDetailsRecord
.getValueByPos(MetadataRecordTypes.FEED_TYPE_SECONDARY_ARECORD_SOURCE_FEED_NAME_FIELD_INDEX))
- .getStringValue();
+ .getStringValue();
- feed = new SecondaryFeed(dataverseName, feedName, sourceFeedName, signature);
+ feed = new Feed(dataverseName, feedName, signature, FeedType.SECONDARY, sourceFeedName, null, null);
}
break;
@@ -215,7 +215,6 @@
switch (feed.getFeedType()) {
case PRIMARY: {
- PrimaryFeed primaryFeed = (PrimaryFeed) feed;
IARecordBuilder primaryDetailsRecordBuilder = new RecordBuilder();
OrderedListBuilder listBuilder = new OrderedListBuilder();
@@ -229,16 +228,16 @@
// write field 0
fieldValue.reset();
- aString.setValue(primaryFeed.getAdaptorName());
+ aString.setValue(feed.getAdapterName());
stringSerde.serialize(aString, primaryRecordfieldValue.getDataOutput());
primaryDetailsRecordBuilder.addField(
MetadataRecordTypes.FEED_ARECORD_PRIMARY_FIELD_DETAILS_ADAPTOR_NAME_FIELD_INDEX,
primaryRecordfieldValue);
// write field 1
- listBuilder
- .reset((AUnorderedListType) MetadataRecordTypes.PRIMARY_FEED_DETAILS_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.FEED_ARECORD_PRIMARY_FIELD_DETAILS_ADAPTOR_CONFIGURATION_FIELD_INDEX]);
- for (Map.Entry<String, String> property : primaryFeed.getAdaptorConfiguration().entrySet()) {
+ listBuilder.reset((AUnorderedListType) MetadataRecordTypes.PRIMARY_FEED_DETAILS_RECORDTYPE
+ .getFieldTypes()[MetadataRecordTypes.FEED_ARECORD_PRIMARY_FIELD_DETAILS_ADAPTOR_CONFIGURATION_FIELD_INDEX]);
+ for (Map.Entry<String, String> property : feed.getAdapterConfiguration().entrySet()) {
String name = property.getKey();
String value = property.getValue();
primaryRecordItemValue.reset();
@@ -262,15 +261,13 @@
break;
case SECONDARY:
- SecondaryFeed secondaryFeed = (SecondaryFeed) feed;
-
IARecordBuilder secondaryDetailsRecordBuilder = new RecordBuilder();
ArrayBackedValueStorage secondaryFieldValue = new ArrayBackedValueStorage();
secondaryDetailsRecordBuilder.reset(MetadataRecordTypes.SECONDARY_FEED_DETAILS_RECORDTYPE);
// write field 0
fieldValue.reset();
- aString.setValue(secondaryFeed.getSourceFeedName());
+ aString.setValue(feed.getSourceFeedName());
stringSerde.serialize(aString, secondaryFieldValue.getDataOutput());
secondaryDetailsRecordBuilder.addField(
MetadataRecordTypes.FEED_ARECORD_SECONDARY_FIELD_DETAILS_SOURCE_FEED_NAME_FIELD_INDEX,
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AbstractDatasourceAdapter.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AbstractDatasourceAdapter.java
deleted file mode 100644
index d65468e..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AbstractDatasourceAdapter.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import java.util.Map;
-
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-
-/**
- * Represents the base class that is required to be extended by every
- * implementation of the IDatasourceAdapter interface.
- */
-public abstract class AbstractDatasourceAdapter implements IDataSourceAdapter {
-
- private static final long serialVersionUID = 1L;
-
- public static final String KEY_PARSER_FACTORY = "parser";
-
- protected Map<String, Object> configuration;
- protected transient AlgebricksPartitionConstraint partitionConstraint;
- protected IAType atype;
- protected IHyracksTaskContext ctx;
-
-}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AbstractFeedDatasourceAdapter.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AbstractFeedDatasourceAdapter.java
deleted file mode 100644
index c231ad9..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AbstractFeedDatasourceAdapter.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.asterix.external.feeds.FeedPolicyEnforcer;
-
-
-public abstract class AbstractFeedDatasourceAdapter implements IDataSourceAdapter {
-
- private static final long serialVersionUID = 1L;
-
- protected FeedPolicyEnforcer policyEnforcer;
-
- public FeedPolicyEnforcer getPolicyEnforcer() {
- return policyEnforcer;
- }
-
- public void setFeedPolicyEnforcer(FeedPolicyEnforcer policyEnforcer) {
- this.policyEnforcer = policyEnforcer;
- }
-
-}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterExecutor.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterExecutor.java
deleted file mode 100644
index 6c2f14c..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterExecutor.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.common.feeds.DistributeFeedFrameWriter;
-import org.apache.asterix.common.feeds.api.IAdapterRuntimeManager;
-import org.apache.asterix.common.feeds.api.IAdapterRuntimeManager.State;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-
-public class AdapterExecutor implements Runnable {
-
- private static final Logger LOGGER = Logger.getLogger(AdapterExecutor.class.getName());
-
- private final DistributeFeedFrameWriter writer;
-
- private final IDataSourceAdapter adapter;
-
- private final IAdapterRuntimeManager adapterManager;
-
- public AdapterExecutor(int partition, DistributeFeedFrameWriter writer, IDataSourceAdapter adapter,
- IAdapterRuntimeManager adapterManager) {
- this.writer = writer;
- this.adapter = adapter;
- this.adapterManager = adapterManager;
- }
-
- @Override
- public void run() {
- int partition = adapterManager.getPartition();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Starting ingestion for partition:" + partition);
- }
- boolean continueIngestion = true;
- boolean failedIngestion = false;
- while (continueIngestion) {
- try {
- adapter.start(partition, writer);
- continueIngestion = false;
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.SEVERE)) {
- LOGGER.severe("Exception during feed ingestion " + e.getMessage());
- e.printStackTrace();
- }
- continueIngestion = adapter.handleException(e);
- failedIngestion = !continueIngestion;
- }
- }
-
- adapterManager.setState(failedIngestion ? State.FAILED_INGESTION : State.FINISHED_INGESTION);
- synchronized (adapterManager) {
- adapterManager.notifyAll();
- }
- }
-
-}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterRuntimeManager.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterRuntimeManager.java
deleted file mode 100644
index aacb3da..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/AdapterRuntimeManager.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.common.feeds.DistributeFeedFrameWriter;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.IngestionRuntime;
-import org.apache.asterix.common.feeds.api.IAdapterRuntimeManager;
-import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-
-public class AdapterRuntimeManager implements IAdapterRuntimeManager {
-
- private static final Logger LOGGER = Logger.getLogger(AdapterRuntimeManager.class.getName());
-
- private final FeedId feedId;
-
- private final IDataSourceAdapter feedAdapter;
-
- private final IIntakeProgressTracker tracker;
-
- private final AdapterExecutor adapterExecutor;
-
- private final int partition;
-
- private final ExecutorService executorService;
-
- private IngestionRuntime ingestionRuntime;
-
- private State state;
-
- public AdapterRuntimeManager(FeedId feedId, IDataSourceAdapter feedAdapter, IIntakeProgressTracker tracker,
- DistributeFeedFrameWriter writer, int partition) {
- this.feedId = feedId;
- this.feedAdapter = feedAdapter;
- this.tracker = tracker;
- this.partition = partition;
- this.adapterExecutor = new AdapterExecutor(partition, writer, feedAdapter, this);
- this.executorService = Executors.newSingleThreadExecutor();
- this.state = State.INACTIVE_INGESTION;
- }
-
- @Override
- public void start() throws Exception {
- state = State.ACTIVE_INGESTION;
- executorService.execute(adapterExecutor);
- }
-
- @Override
- public void stop() {
- try {
- feedAdapter.stop();
- } catch (Exception exception) {
- if (LOGGER.isLoggable(Level.SEVERE)) {
- LOGGER.severe("Unable to stop adapter " + feedAdapter + ", encountered exception " + exception);
- }
- } finally {
- state = State.FINISHED_INGESTION;
- executorService.shutdown();
- }
- }
-
- @Override
- public FeedId getFeedId() {
- return feedId;
- }
-
- @Override
- public String toString() {
- return feedId + "[" + partition + "]";
- }
-
- @Override
- public IDataSourceAdapter getFeedAdapter() {
- return feedAdapter;
- }
-
- public IIntakeProgressTracker getTracker() {
- return tracker;
- }
-
- @Override
- public synchronized State getState() {
- return state;
- }
-
- @Override
- public synchronized void setState(State state) {
- this.state = state;
- }
-
- public AdapterExecutor getAdapterExecutor() {
- return adapterExecutor;
- }
-
- @Override
- public int getPartition() {
- return partition;
- }
-
- public IngestionRuntime getIngestionRuntime() {
- return ingestionRuntime;
- }
-
- @Override
- public IIntakeProgressTracker getProgressTracker() {
- return tracker;
- }
-
-}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java
index a144d5f..8ef6732 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java
@@ -21,37 +21,37 @@
import java.util.HashMap;
import java.util.Map;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.common.config.MetadataConstants;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
public class BuiltinFeedPolicies {
- public static final FeedPolicy BRITTLE = initializeBrittlePolicy();
+ public static final FeedPolicyEntity BRITTLE = initializeBrittlePolicy();
- public static final FeedPolicy BASIC = initializeBasicPolicy();
+ public static final FeedPolicyEntity BASIC = initializeBasicPolicy();
- public static final FeedPolicy BASIC_FT = initializeBasicFTPolicy();
+ public static final FeedPolicyEntity BASIC_FT = initializeBasicFTPolicy();
- public static final FeedPolicy ADVANCED_FT = initializeAdvancedFTPolicy();
+ public static final FeedPolicyEntity ADVANCED_FT = initializeAdvancedFTPolicy();
- public static final FeedPolicy ADVANCED_FT_DISCARD = initializeAdvancedFTDiscardPolicy();
+ public static final FeedPolicyEntity ADVANCED_FT_DISCARD = initializeAdvancedFTDiscardPolicy();
- public static final FeedPolicy ADVANCED_FT_SPILL = initializeAdvancedFTSpillPolicy();
+ public static final FeedPolicyEntity ADVANCED_FT_SPILL = initializeAdvancedFTSpillPolicy();
- public static final FeedPolicy ADVANCED_FT_THROTTLE = initializeAdvancedFTThrottlePolicy();
+ public static final FeedPolicyEntity ADVANCED_FT_THROTTLE = initializeAdvancedFTThrottlePolicy();
- public static final FeedPolicy ELASTIC = initializeAdvancedFTElasticPolicy();
+ public static final FeedPolicyEntity ELASTIC = initializeAdvancedFTElasticPolicy();
- public static final FeedPolicy[] policies = new FeedPolicy[] { BRITTLE, BASIC, BASIC_FT, ADVANCED_FT,
+ public static final FeedPolicyEntity[] policies = new FeedPolicyEntity[] { BRITTLE, BASIC, BASIC_FT, ADVANCED_FT,
ADVANCED_FT_DISCARD, ADVANCED_FT_SPILL, ADVANCED_FT_THROTTLE, ELASTIC };
- public static final FeedPolicy DEFAULT_POLICY = BASIC_FT;
+ public static final FeedPolicyEntity DEFAULT_POLICY = BASIC_FT;
public static final String CONFIG_FEED_POLICY_KEY = "policy";
- public static FeedPolicy getFeedPolicy(String policyName) {
- for (FeedPolicy policy : policies) {
+ public static FeedPolicyEntity getFeedPolicy(String policyName) {
+ for (FeedPolicyEntity policy : policies) {
if (policy.getPolicyName().equalsIgnoreCase(policyName)) {
return policy;
}
@@ -60,7 +60,7 @@
}
//Brittle
- private static FeedPolicy initializeBrittlePolicy() {
+ private static FeedPolicyEntity initializeBrittlePolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "false");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "false");
@@ -71,11 +71,11 @@
policyParams.put(FeedPolicyAccessor.AT_LEAST_ONE_SEMANTICS, "false");
String description = "Brittle";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "Brittle", description, policyParams);
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Brittle", description, policyParams);
}
//Basic
- private static FeedPolicy initializeBasicPolicy() {
+ private static FeedPolicyEntity initializeBasicPolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "false");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "true");
@@ -85,11 +85,11 @@
policyParams.put(FeedPolicyAccessor.AT_LEAST_ONE_SEMANTICS, "false");
String description = "Basic";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "Basic", description, policyParams);
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Basic", description, policyParams);
}
// BasicFT
- private static FeedPolicy initializeBasicFTPolicy() {
+ private static FeedPolicyEntity initializeBasicFTPolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "true");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "true");
@@ -103,11 +103,11 @@
policyParams.put(FeedPolicyAccessor.THROTTLING_ENABLED, "false");
String description = "Basic Monitored Fault-Tolerant";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "BasicFT", description, policyParams);
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "BasicFT", description, policyParams);
}
// AdvancedFT
- private static FeedPolicy initializeAdvancedFTPolicy() {
+ private static FeedPolicyEntity initializeAdvancedFTPolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "true");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "true");
@@ -118,11 +118,11 @@
policyParams.put(FeedPolicyAccessor.AT_LEAST_ONE_SEMANTICS, "true");
String description = "Basic Monitored Fault-Tolerant with at least once semantics";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT", description, policyParams);
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT", description, policyParams);
}
// AdvancedFT_Discard
- private static FeedPolicy initializeAdvancedFTDiscardPolicy() {
+ private static FeedPolicyEntity initializeAdvancedFTDiscardPolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "true");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "true");
@@ -133,14 +133,14 @@
policyParams.put(FeedPolicyAccessor.MAX_FRACTION_DISCARD, "100");
policyParams.put(FeedPolicyAccessor.TIME_TRACKING, "false");
policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, "true");
-
+
String description = "AdvancedFT 100% Discard during congestion";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Discard", description,
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Discard", description,
policyParams);
}
// AdvancedFT_Spill
- private static FeedPolicy initializeAdvancedFTSpillPolicy() {
+ private static FeedPolicyEntity initializeAdvancedFTSpillPolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "true");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "true");
@@ -152,11 +152,11 @@
policyParams.put(FeedPolicyAccessor.TIME_TRACKING, "true");
String description = "AdvancedFT 100% Discard during congestion";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Spill", description, policyParams);
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Spill", description, policyParams);
}
// AdvancedFT_Spill
- private static FeedPolicy initializeAdvancedFTThrottlePolicy() {
+ private static FeedPolicyEntity initializeAdvancedFTThrottlePolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "true");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "true");
@@ -169,12 +169,12 @@
policyParams.put(FeedPolicyAccessor.THROTTLING_ENABLED, "true");
String description = "AdvancedFT Throttle during congestion";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Throttle", description,
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Throttle", description,
policyParams);
}
// AdvancedFT_Elastic
- private static FeedPolicy initializeAdvancedFTElasticPolicy() {
+ private static FeedPolicyEntity initializeAdvancedFTElasticPolicy() {
Map<String, String> policyParams = new HashMap<String, String>();
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_CONTINUE, "true");
policyParams.put(FeedPolicyAccessor.SOFT_FAILURE_LOG_DATA, "true");
@@ -185,7 +185,7 @@
policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, "true");
String description = "Basic Monitored Fault-Tolerant Elastic";
- return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Elastic", description,
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Elastic", description,
policyParams);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedActivityIdFactory.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedActivityIdFactory.java
deleted file mode 100644
index a0a4af9..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedActivityIdFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-public class FeedActivityIdFactory {
- private static AtomicInteger id = new AtomicInteger();
- private static boolean isInitialized = false;
-
- public static boolean isInitialized() {
- return isInitialized;
- }
-
- public static void initialize(int initialId) {
- id.set(initialId);
- isInitialized = true;
- }
-
- public static int generateFeedActivityId() {
- return id.incrementAndGet();
- }
-
-}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedUtil.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
similarity index 91%
rename from asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedUtil.java
rename to asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 5ed2876..5b5f19f 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedUtil.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -30,31 +30,32 @@
import java.util.logging.Logger;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.dataflow.AsterixLSMInvertedIndexInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.FeedRuntimeId;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.api.IDataSourceAdapter;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.runtime.FeedRuntimeId;
import org.apache.asterix.external.library.ExternalLibraryManager;
+import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
+import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor;
import org.apache.asterix.external.provider.AdapterFactoryProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
-import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
-import org.apache.asterix.metadata.entities.SecondaryFeed;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
@@ -85,23 +86,11 @@
/**
* A utility class for providing helper functions for feeds
+ * TODO: Refactor this class.
*/
-public class FeedUtil {
+public class FeedMetadataUtil {
- private static Logger LOGGER = Logger.getLogger(FeedUtil.class.getName());
-
- public static String getFeedPointKeyRep(Feed feed, List<String> appliedFunctions) {
- StringBuilder builder = new StringBuilder();
- builder.append(feed.getDataverseName() + ":");
- builder.append(feed.getFeedName() + ":");
- if (appliedFunctions != null && !appliedFunctions.isEmpty()) {
- for (String function : appliedFunctions) {
- builder.append(function + ":");
- }
- builder.deleteCharAt(builder.length() - 1);
- }
- return builder.toString();
- }
+ private static Logger LOGGER = Logger.getLogger(FeedMetadataUtil.class.getName());
private static class LocationConstraint {
int partition;
@@ -131,9 +120,9 @@
return feed;
}
- public static FeedPolicy validateIfPolicyExists(String dataverse, String policyName, MetadataTransactionContext ctx)
- throws AsterixException {
- FeedPolicy feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(ctx, dataverse, policyName);
+ public static FeedPolicyEntity validateIfPolicyExists(String dataverse, String policyName,
+ MetadataTransactionContext ctx) throws AsterixException {
+ FeedPolicyEntity feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(ctx, dataverse, policyName);
if (feedPolicy == null) {
feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(ctx, MetadataConstants.METADATA_DATAVERSE_NAME,
policyName);
@@ -466,26 +455,31 @@
return preProcessingRequired;
}
- public static Triple<IAdapterFactory, ARecordType, AdapterType> getPrimaryFeedFactoryAndOutput(PrimaryFeed feed,
- FeedPolicyAccessor policyAccessor, MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
-
+ public static Triple<IAdapterFactory, ARecordType, IDataSourceAdapter.AdapterType> getPrimaryFeedFactoryAndOutput(
+ Feed feed, FeedPolicyAccessor policyAccessor, MetadataTransactionContext mdTxnCtx)
+ throws AlgebricksException {
+ // This method needs to be re-visited
String adapterName = null;
DatasourceAdapter adapterEntity = null;
String adapterFactoryClassname = null;
IAdapterFactory adapterFactory = null;
ARecordType adapterOutputType = null;
- Triple<IAdapterFactory, ARecordType, AdapterType> feedProps = null;
- AdapterType adapterType = null;
+ Triple<IAdapterFactory, ARecordType, IDataSourceAdapter.AdapterType> feedProps = null;
+ IDataSourceAdapter.AdapterType adapterType = null;
try {
- adapterName = feed.getAdaptorName();
- Map<String, String> configuration = feed.getAdaptorConfiguration();
+ adapterName = feed.getAdapterName();
+ Map<String, String> configuration = feed.getAdapterConfiguration();
configuration.putAll(policyAccessor.getFeedPolicy());
adapterOutputType = getOutputType(feed, configuration);
+ ExternalDataUtils.prepareFeed(configuration, feed.getDataverseName(), feed.getFeedName());
+ // Get adapter from metadata dataset <Metadata dataverse>
adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME,
adapterName);
+ // Get adapter from metadata dataset <The feed dataverse>
if (adapterEntity == null) {
adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, feed.getDataverseName(), adapterName);
}
+
if (adapterEntity != null) {
adapterType = adapterEntity.getType();
adapterFactoryClassname = adapterEntity.getClassname();
@@ -503,13 +497,12 @@
}
adapterFactory.configure(configuration, adapterOutputType);
} else {
- configuration.put(ExternalDataConstants.KEY_DATAVERSE, feed.getDataverseName());
adapterFactory = AdapterFactoryProvider.getAdapterFactory(adapterName, configuration,
adapterOutputType);
- adapterType = AdapterType.INTERNAL;
+ adapterType = IDataSourceAdapter.AdapterType.INTERNAL;
}
- feedProps = new Triple<IAdapterFactory, ARecordType, AdapterType>(adapterFactory, adapterOutputType,
- adapterType);
+ feedProps = new Triple<IAdapterFactory, ARecordType, IDataSourceAdapter.AdapterType>(adapterFactory,
+ adapterOutputType, adapterType);
} catch (Exception e) {
e.printStackTrace();
throw new AlgebricksException("unable to create adapter " + e);
@@ -517,7 +510,7 @@
return feedProps;
}
- private static ARecordType getOutputType(PrimaryFeed feed, Map<String, String> configuration) throws Exception {
+ private static ARecordType getOutputType(Feed feed, Map<String, String> configuration) throws Exception {
ARecordType outputType = null;
String fqOutputType = configuration.get(ExternalDataConstants.KEY_TYPE_NAME);
@@ -560,15 +553,15 @@
return outputType;
}
- public static String getSecondaryFeedOutput(SecondaryFeed feed, FeedPolicyAccessor policyAccessor,
+ public static String getSecondaryFeedOutput(Feed feed, FeedPolicyAccessor policyAccessor,
MetadataTransactionContext mdTxnCtx) throws AlgebricksException, MetadataException {
String outputType = null;
String primaryFeedName = feed.getSourceFeedName();
Feed primaryFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, feed.getDataverseName(), primaryFeedName);
FunctionSignature appliedFunction = primaryFeed.getAppliedFunction();
if (appliedFunction == null) {
- Triple<IAdapterFactory, ARecordType, AdapterType> result = getPrimaryFeedFactoryAndOutput(
- (PrimaryFeed) primaryFeed, policyAccessor, mdTxnCtx);
+ Triple<IAdapterFactory, ARecordType, IDataSourceAdapter.AdapterType> result = getPrimaryFeedFactoryAndOutput(
+ primaryFeed, policyAccessor, mdTxnCtx);
outputType = result.second.getTypeName();
} else {
Function function = MetadataManager.INSTANCE.getFunction(mdTxnCtx, appliedFunction);
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/IAdapterExecutor.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/IAdapterExecutor.java
deleted file mode 100644
index ff641af..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/IAdapterExecutor.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import org.apache.asterix.common.feeds.FeedConnectionId;
-
-public interface IAdapterExecutor {
-
- /**
- * @throws Exception
- */
- public void start() throws Exception;
-
- /**
- * @throws Exception
- */
- public void stop() throws Exception;
-
- /**
- * @return
- */
- public FeedConnectionId getFeedId();
-
-}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/IFeedMessage.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/IFeedMessage.java
deleted file mode 100644
index 9180671..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/IFeedMessage.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import java.io.Serializable;
-
-public interface IFeedMessage extends Serializable {
-
- public enum MessageType {
- END,
- SUPER_FEED_MANAGER_ELECT
- }
-
- public MessageType getMessageType();
-
-}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/ITypedAdapterFactory.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/ITypedAdapterFactory.java
deleted file mode 100644
index f35c21f..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/ITypedAdapterFactory.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import java.util.Map;
-
-import org.apache.asterix.external.api.IAdapterFactory;
-import org.apache.asterix.om.types.ARecordType;
-
-public interface ITypedAdapterFactory extends IAdapterFactory {
-
- public ARecordType getAdapterOutputType();
-
- public void configure(Map<String, String> configuration) throws Exception;
-}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/MessageListener.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/MessageListener.java
deleted file mode 100644
index 650cb92..0000000
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/MessageListener.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.feeds;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.nio.CharBuffer;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class MessageListener {
-
- private static final Logger LOGGER = Logger.getLogger(MessageListener.class.getName());
-
- private final int port;
- private final LinkedBlockingQueue<String> outbox;
-
- private ExecutorService executorService = Executors.newFixedThreadPool(10);
-
- private MessageListenerServer listenerServer;
-
- public MessageListener(int port, LinkedBlockingQueue<String> outbox) {
- this.port = port;
- this.outbox = outbox;
- }
-
- public void stop() {
- listenerServer.stop();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Stopped message service at " + port);
- }
- if (!executorService.isShutdown()) {
- executorService.shutdownNow();
- }
-
- }
-
- public void start() throws IOException {
- listenerServer = new MessageListenerServer(port, outbox);
- executorService.execute(listenerServer);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Starting message service at " + port);
- }
- }
-
- private static class MessageListenerServer implements Runnable {
-
- private final int port;
- private final LinkedBlockingQueue<String> outbox;
- private ServerSocket server;
-
- public MessageListenerServer(int port, LinkedBlockingQueue<String> outbox) {
- this.port = port;
- this.outbox = outbox;
- }
-
- public void stop() {
- try {
- server.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void run() {
- char EOL = (char) "\n".getBytes()[0];
- Socket client = null;
- try {
- server = new ServerSocket(port);
- client = server.accept();
- InputStream in = client.getInputStream();
- CharBuffer buffer = CharBuffer.allocate(5000);
- char ch;
- while (true) {
- ch = (char) in.read();
- if (((int) ch) == -1) {
- break;
- }
- while (ch != EOL) {
- buffer.put(ch);
- ch = (char) in.read();
- }
- buffer.flip();
- String s = new String(buffer.array());
- synchronized (outbox) {
- outbox.add(s + "\n");
- }
- buffer.position(0);
- buffer.limit(5000);
- }
-
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Unable to start Message listener" + server);
- }
- } finally {
- if (server != null) {
- try {
- server.close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
-
- }
-
- }
-
- public static interface IMessageAnalyzer {
-
- public LinkedBlockingQueue<String> getMessageQueue();
-
- }
-
-}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
index f7a0e5d..f011f04 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
@@ -26,6 +26,7 @@
import org.apache.asterix.builders.IARecordBuilder;
import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -35,7 +36,6 @@
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.asterix.metadata.entities.CompactionPolicy;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
index 7c32bdf..b6f3c9e 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
@@ -23,40 +23,34 @@
import java.util.List;
import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.asterix.common.config.MetadataConstants;
+import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.om.util.AsterixClusterProperties;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
public class SplitsAndConstraintsUtil {
- public static final String PARTITION_DIR_PREFIX = "partition_";
- public static final String TEMP_DATASETS_STORAGE_FOLDER = "temp";
- public static final String DATASET_INDEX_NAME_SEPARATOR = "_idx_";
-
private static FileSplit[] splitsForDataverse(String dataverseName) {
File relPathFile = new File(dataverseName);
List<FileSplit> splits = new ArrayList<FileSplit>();
- //get all partitions
+ // get all partitions
ClusterPartition[] clusterPartition = AsterixClusterProperties.INSTANCE.getClusterPartitons();
String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
for (int j = 0; j < clusterPartition.length; j++) {
int nodeParitions = AsterixClusterProperties.INSTANCE
.getNodePartitionsCount(clusterPartition[j].getNodeId());
for (int i = 0; i < nodeParitions; i++) {
- File f = new File(prepareStoragePartitionPath(storageDirName, clusterPartition[i].getPartitionId())
- + File.separator + relPathFile);
- splits.add(getFileSplitForClusterPartition(clusterPartition[j], f));
+ File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
+ clusterPartition[i].getPartitionId()) + File.separator + relPathFile);
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition[j], f));
}
}
return splits.toArray(new FileSplit[] {});
@@ -65,7 +59,8 @@
public static FileSplit[] splitsForDataset(MetadataTransactionContext mdTxnCtx, String dataverseName,
String datasetName, String targetIdxName, boolean temp) throws AlgebricksException {
try {
- File relPathFile = new File(prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
+ File relPathFile = new File(
+ StoragePathUtil.prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
List<String> nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName())
.getNodeNames();
@@ -78,18 +73,18 @@
for (String nd : nodeGroup) {
int numPartitions = AsterixClusterProperties.INSTANCE.getNodePartitionsCount(nd);
ClusterPartition[] nodePartitions = AsterixClusterProperties.INSTANCE.getNodePartitions(nd);
- //currently this case is never executed since the metadata group doesn't exists
+ // currently this case is never executed since the metadata group doesn't exists
if (dataset.getNodeGroupName().compareTo(MetadataConstants.METADATA_NODEGROUP_NAME) == 0) {
numPartitions = 1;
}
for (int k = 0; k < numPartitions; k++) {
- //format: 'storage dir name'/partition_#/dataverse/dataset_idx_index
- //temp format: 'storage dir name'/temp/partition_#/dataverse/dataset_idx_index
- File f = new File(prepareStoragePartitionPath(
- storageDirName + (temp ? (File.separator + TEMP_DATASETS_STORAGE_FOLDER) : ""),
- nodePartitions[k].getPartitionId()) + File.separator + relPathFile);
- splits.add(getFileSplitForClusterPartition(nodePartitions[k], f));
+ // format: 'storage dir name'/partition_#/dataverse/dataset_idx_index
+ File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
+ nodePartitions[k].getPartitionId())
+ + (temp ? (File.separator + StoragePathUtil.TEMP_DATASETS_STORAGE_FOLDER) : "")
+ + File.separator + relPathFile);
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[k], f));
}
}
return splits.toArray(new FileSplit[] {});
@@ -101,7 +96,8 @@
private static FileSplit[] splitsForFilesIndex(MetadataTransactionContext mdTxnCtx, String dataverseName,
String datasetName, String targetIdxName, boolean create) throws AlgebricksException {
try {
- File relPathFile = new File(prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
+ File relPathFile = new File(
+ StoragePathUtil.prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
List<String> nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName())
.getNodeNames();
@@ -111,21 +107,20 @@
List<FileSplit> splits = new ArrayList<FileSplit>();
for (String nodeId : nodeGroup) {
- //get node partitions
+ // get node partitions
ClusterPartition[] nodePartitions = AsterixClusterProperties.INSTANCE.getNodePartitions(nodeId);
String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
int firstPartition = 0;
if (create) {
// Only the first partition when create
- File f = new File(
- prepareStoragePartitionPath(storageDirName, nodePartitions[firstPartition].getPartitionId())
- + File.separator + relPathFile);
- splits.add(getFileSplitForClusterPartition(nodePartitions[firstPartition], f));
+ File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
+ nodePartitions[firstPartition].getPartitionId()) + File.separator + relPathFile);
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[firstPartition], f));
} else {
for (int k = 0; k < nodePartitions.length; k++) {
- File f = new File(prepareStoragePartitionPath(storageDirName,
+ File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
nodePartitions[firstPartition].getPartitionId()) + File.separator + relPathFile);
- splits.add(getFileSplitForClusterPartition(nodePartitions[firstPartition], f));
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[firstPartition], f));
}
}
}
@@ -138,37 +133,13 @@
public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataverse(
String dataverse) {
FileSplit[] splits = splitsForDataverse(dataverse);
- return splitProviderAndPartitionConstraints(splits);
+ return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForFilesIndex(
MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName, String targetIdxName,
boolean create) throws AlgebricksException {
FileSplit[] splits = splitsForFilesIndex(mdTxnCtx, dataverseName, datasetName, targetIdxName, create);
- return splitProviderAndPartitionConstraints(splits);
- }
-
- public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraints(
- FileSplit[] splits) {
- IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
- String[] loc = new String[splits.length];
- for (int p = 0; p < splits.length; p++) {
- loc[p] = splits[p].getNodeName();
- }
- AlgebricksPartitionConstraint pc = new AlgebricksAbsolutePartitionConstraint(loc);
- return new Pair<IFileSplitProvider, AlgebricksPartitionConstraint>(splitProvider, pc);
- }
-
- private static FileSplit getFileSplitForClusterPartition(ClusterPartition partition, File relativeFile) {
- return new FileSplit(partition.getActiveNodeId(), new FileReference(relativeFile), partition.getIODeviceNum(),
- partition.getPartitionId());
- }
-
- public static String prepareStoragePartitionPath(String storageDirName, int partitonId) {
- return storageDirName + File.separator + PARTITION_DIR_PREFIX + partitonId;
- }
-
- private static String prepareDataverseIndexName(String dataverseName, String datasetName, String idxName) {
- return dataverseName + File.separator + datasetName + DATASET_INDEX_NAME_SEPARATOR + idxName;
+ return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
}
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java b/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java
index 95eea63..80008c5 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java
@@ -77,7 +77,7 @@
} else {
cluster = null;
}
- //if this is the CC process
+ // if this is the CC process
if (AsterixAppContextInfo.getInstance() != null) {
if (AsterixAppContextInfo.getInstance().getCCApplicationContext() != null) {
node2PartitionsMap = AsterixAppContextInfo.getInstance().getMetadataProperties().getNodePartitions();
@@ -94,11 +94,11 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(" Removing configuration parameters for node id " + nodeId);
}
- //TODO implement fault tolerance as follows:
- //1. collect the partitions of the failed NC
- //2. For each partition, request a remote replica to take over.
- //3. wait until each remote replica completes the recovery for the lost partitions
- //4. update the cluster state
+ // TODO implement fault tolerance as follows:
+ // 1. collect the partitions of the failed NC
+ // 2. For each partition, request a remote replica to take over.
+ // 3. wait until each remote replica completes the recovery for the lost partitions
+ // 4. update the cluster state
}
public synchronized void addNCConfiguration(String nodeId, Map<String, String> configuration) {
@@ -111,10 +111,10 @@
private synchronized void updateNodePartitions(String nodeId, boolean added) {
ClusterPartition[] nodePartitions = node2PartitionsMap.get(nodeId);
- //if this isn't a storage node, it will not have cluster partitions
+ // if this isn't a storage node, it will not have cluster partitions
if (nodePartitions != null) {
for (ClusterPartition p : nodePartitions) {
- //set the active node for this node's partitions
+ // set the active node for this node's partitions
p.setActive(added);
if (added) {
p.setActiveNodeId(nodeId);
@@ -135,14 +135,13 @@
return;
}
}
- //if all storage partitions are active, then the cluster is active
+ // if all storage partitions are active, then the cluster is active
state = ClusterState.ACTIVE;
LOGGER.info("Cluster is now ACTIVE");
}
/**
* Returns the number of IO devices configured for a Node Controller
- *
* @param nodeId
* unique identifier of the Node Controller
* @return number of IO devices. -1 if the node id is not valid. A node id
@@ -156,7 +155,6 @@
/**
* Returns the IO devices configured for a Node Controller
- *
* @param nodeId
* unique identifier of the Node Controller
* @return a list of IO devices. null if node id is not valid. A node id is not valid
@@ -223,7 +221,7 @@
public static boolean isClusterActive() {
if (AsterixClusterProperties.INSTANCE.getCluster() == null) {
- //this is a virtual cluster
+ // this is a virtual cluster
return true;
}
return AsterixClusterProperties.INSTANCE.getState() == ClusterState.ACTIVE;
@@ -256,7 +254,7 @@
if (cluster != null) {
return cluster.getStore();
}
- //virtual cluster without cluster config file
+ // virtual cluster without cluster config file
return DEFAULT_STORAGE_DIR_NAME;
}
}