Cleanup FileSplit and FileReference
This change gives FileSplit and FileReference specific meaning to
avoid confusion of an absolute vs relative, local vs global, inside
an IO device vs outside IO devices.
In addition, it enables better abstraction of global partitions and
delegate the responsibility of choosing which partition goes to which
IO device to the IO Manager through the introduction of FileDeviceComputer
In details:
Previously, the LocalResource in Hyracks had partition (storage partition)
and there is no such thing in Hyracks. This scope leak is bad. In addition
The local resource had a name and a path. they were always the same and so
the name was removed.
The storage partition was instead moved to asterixdb implementation of the
serialized object in the local resource.
With all of these changes, the cluster controller (compiler) only needs to
know about partitions and relative paths. It doesn't need to worry about
heterogenous Node setups and different io device configurations. For File
assignment to IO devices, a new interface (IFileDeviceComputer) was
introduced which can be overriden by applications to have their own
strategy for distributing files among IO devices.
Change-Id: I4fac508bf9af5a3bed41a3cf4464d2cbfecf2f61
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1352
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitPOperator.java
index 5426ccc..28c883a 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitPOperator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitPOperator.java
@@ -37,7 +37,7 @@
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class CommitPOperator extends AbstractPhysicalOperator {
@@ -97,7 +97,7 @@
dataverse, dataset, dataset, metadataProvider.isTemporaryDatasetWriteJob());
int[] datasetPartitions = new int[splitsForDataset.length];
for (int i = 0; i < splitsForDataset.length; i++) {
- datasetPartitions[i] = splitsForDataset[i].getPartition();
+ datasetPartitions[i] = i;
}
int upsertVarIdx = -1;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 8a1fb3f..fc982d8 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.translator;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -157,8 +156,7 @@
import org.apache.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
import org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
/**
* Each visit returns a pair of an operator and a variable. The variable
@@ -702,7 +700,7 @@
InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
- insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+ insertOp.getInputs().add(new MutableObject<>(assign));
CompiledInsertStatement compiledInsert = (CompiledInsertStatement) stmt;
if (compiledInsert.getReturnQuery() != null) {
leafOperator = createReturningQuery(compiledInsert, insertOp);
@@ -737,7 +735,7 @@
String filePath = outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX
+ outputFileID.incrementAndGet();
AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.INSTANCE.getMetadataProperties();
- return new FileSplit(metadataProperties.getMetadataNodeName(), new FileReference(new File(filePath)));
+ return new FileSplit(metadataProperties.getMetadataNodeName(), filePath, true);
}
@Override
@@ -1723,7 +1721,7 @@
UnnestOperator unnestOp = new UnnestOperator(unnestVar,
new MutableObject<ILogicalExpression>(new UnnestingFunctionCallExpression(
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.SCAN_COLLECTION), args)));
- unnestOp.getInputs().add(new MutableObject<ILogicalOperator>(opAndVar.first));
+ unnestOp.getInputs().add(new MutableObject<>(opAndVar.first));
inputOpRefsToUnion.add(new MutableObject<ILogicalOperator>(unnestOp));
vars.add(unnestVar);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
index 3562ba9..7982f12 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
@@ -40,7 +40,7 @@
import org.apache.asterix.util.FlushDatasetUtils;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.client.NodeControllerInfo;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -139,8 +139,9 @@
// Generates file partitions.
for (FileSplit split : fileSplits) {
String ipAddress = nodeMap.get(split.getNodeName()).getNetworkAddress().getAddress().toString();
- String path = split.getLocalFile().getFile().getAbsolutePath();
- FilePartition partition = new FilePartition(ipAddress, path, split.getIODeviceId());
+ String path = split.getPath();
+ boolean relative = split.isManaged();
+ FilePartition partition = new FilePartition(ipAddress, path, relative);
partititons.put(partition.toJSONObject());
}
// Generates the response object which contains the splits.
@@ -151,12 +152,12 @@
class FilePartition {
private final String ipAddress;
private final String path;
- private final int ioDeviceId;
+ private final boolean relative;
- public FilePartition(String ipAddress, String path, int ioDeviceId) {
+ public FilePartition(String ipAddress, String path, boolean relative) {
this.ipAddress = ipAddress;
this.path = path;
- this.ioDeviceId = ioDeviceId;
+ this.relative = relative;
}
public String getIPAddress() {
@@ -167,8 +168,8 @@
return path;
}
- public int getIODeviceId() {
- return ioDeviceId;
+ public boolean isRelative() {
+ return relative;
}
@Override
@@ -180,7 +181,7 @@
JSONObject partition = new JSONObject();
partition.put("ip", ipAddress);
partition.put("path", path);
- partition.put("ioDeviceId", ioDeviceId);
+ partition.put("relative", relative);
return partition;
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
index d4b6744..3e9c34f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
@@ -26,7 +26,6 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
@@ -38,6 +37,7 @@
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
+import org.apache.asterix.common.transactions.IResourceFactory;
import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.indexing.ExternalFile;
@@ -72,7 +72,7 @@
import org.apache.asterix.runtime.util.AsterixAppContextInfo;
import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
-import org.apache.asterix.transaction.management.resource.ExternalBTreeLocalResourceMetadata;
+import org.apache.asterix.transaction.management.resource.ExternalBTreeLocalResourceMetadataFactory;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import org.apache.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
@@ -224,7 +224,7 @@
getFilesIndexName(dataset.getDatasetName()), true);
IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
FilesIndexDescription filesIndexDescription = new FilesIndexDescription();
- ILocalResourceMetadata localResourceMetadata = new ExternalBTreeLocalResourceMetadata(
+ IResourceFactory localResourceMetadata = new ExternalBTreeLocalResourceMetadataFactory(
filesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS, filesIndexDescription.FILES_INDEX_COMP_FACTORIES,
new int[] { 0 }, false, dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties);
PersistentLocalResourceFactoryProvider localResourceFactoryProvider =
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
index 4018690..66a55c7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
@@ -50,10 +50,10 @@
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.FileRemoveOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java
index 3447648..9818116 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java
@@ -80,6 +80,7 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
@@ -124,7 +125,7 @@
private ILSMIOOperationScheduler lsmIOScheduler;
private PersistentLocalResourceRepository localResourceRepository;
- private IIOManager ioManager;
+ private IOManager ioManager;
private boolean isShuttingdown;
private ActiveManager activeManager;
@@ -167,11 +168,11 @@
Logger.getLogger("org.apache.asterix").setLevel(externalProperties.getLogLevel());
Logger.getLogger("org.apache.hyracks").setLevel(externalProperties.getLogLevel());
+ ioManager = (IOManager) ncApplicationContext.getIoManager();
threadExecutor = new AsterixThreadExecutor(ncApplicationContext.getThreadFactory());
- fileMapManager = new AsterixFileMapManager();
+ fileMapManager = new AsterixFileMapManager(ioManager);
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
- ioManager = ncApplicationContext.getIoManager();
IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator,
storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index bec419a..36adb1f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -197,10 +197,9 @@
import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.json.JSONArray;
import org.json.JSONException;
@@ -420,7 +419,7 @@
throws InstantiationException, IllegalAccessException, ClassNotFoundException {
WriteStatement ws = (WriteStatement) stmt;
File f = new File(ws.getFileName());
- FileSplit outputFile = new FileSplit(ws.getNcName().getValue(), new FileReference(f));
+ FileSplit outputFile = new FileSplit(ws.getNcName().getValue(), f.getPath(), false);
IAWriterFactory writerFactory = null;
if (ws.getWriterClassName() != null) {
writerFactory = (IAWriterFactory) Class.forName(ws.getWriterClassName()).newInstance();
@@ -3178,7 +3177,7 @@
int j;
// Build the stack for the enforced type
for (j = 1; j < splits.size(); j++) {
- nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
+ nestedTypeStack.push(new Pair<>(nestedFieldType, splits.get(j - 1)));
bridgeName = nestedFieldType.getTypeName();
nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
if (nestedFieldType == null) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
index 6ab2622..675cb80 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
@@ -24,13 +24,13 @@
import java.util.Map;
import java.util.logging.Logger;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import org.apache.asterix.common.transactions.IResourceFactory;
import org.apache.asterix.formats.base.IDataFormat;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -41,7 +41,7 @@
import org.apache.asterix.runtime.util.AsterixAppContextInfo;
import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
-import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
+import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadataFactory;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import org.apache.asterix.translator.CompiledStatements.CompiledDatasetDropStatement;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
@@ -51,8 +51,8 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
@@ -182,7 +182,7 @@
metadata.getMetadataTxnContext());
AsterixStorageProperties storageProperties = AsterixAppContextInfo.INSTANCE.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
- ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
+ IResourceFactory localResourceMetadata = new LSMBTreeLocalResourceMetadataFactory(typeTraits,
comparatorFactories, bloomFilterKeyFields, true, dataset.getDatasetId(), compactionInfo.first,
compactionInfo.second, filterTypeTraits, filterCmpFactories, btreeFields, filterFields);
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
@@ -205,7 +205,7 @@
}
private static String stringOf(FileSplit fs) {
- return fs.getNodeName() + ":" + fs.getLocalFile().toString();
+ return fs.getNodeName() + ":" + fs.getPath();
}
public static JobSpecification compactDatasetJobSpec(Dataverse dataverse, String datasetName,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
index 84e5e39..459f20e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -20,7 +20,6 @@
import java.util.List;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -30,7 +29,6 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
-import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -38,9 +36,10 @@
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
-import org.apache.asterix.transaction.management.resource.ExternalBTreeWithBuddyLocalResourceMetadata;
-import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
+import org.apache.asterix.transaction.management.resource.ExternalBTreeWithBuddyLocalResourceMetadataFactory;
+import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadataFactory;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -91,7 +90,8 @@
IIndexDataflowHelperFactory indexDataflowHelperFactory;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
- ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(secondaryTypeTraits,
+ LSMBTreeLocalResourceMetadataFactory localResourceMetadata = new LSMBTreeLocalResourceMetadataFactory(
+ secondaryTypeTraits,
secondaryComparatorFactories, secondaryBloomFilterKeyFields, false, dataset.getDatasetId(),
mergePolicyFactory, mergePolicyFactoryProperties, filterTypeTraits, filterCmpFactories,
secondaryBTreeFields, secondaryFilterFields);
@@ -106,7 +106,8 @@
} else {
// External dataset local resource and dataflow helper
int[] buddyBreeFields = new int[] { numSecondaryKeys };
- ILocalResourceMetadata localResourceMetadata = new ExternalBTreeWithBuddyLocalResourceMetadata(
+ ExternalBTreeWithBuddyLocalResourceMetadataFactory localResourceMetadata =
+ new ExternalBTreeWithBuddyLocalResourceMetadataFactory(
dataset.getDatasetId(), secondaryComparatorFactories, secondaryTypeTraits, mergePolicyFactory,
mergePolicyFactoryProperties, buddyBreeFields);
localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java
index f452dab..cacf44e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java
@@ -20,13 +20,13 @@
import java.util.List;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.IAsterixPropertiesProvider;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
+import org.apache.asterix.common.transactions.IResourceFactory;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.types.IAType;
@@ -34,7 +34,7 @@
import org.apache.asterix.runtime.formats.FormatUtils;
import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
-import org.apache.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadata;
+import org.apache.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadataFactory;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -217,7 +217,7 @@
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
- ILocalResourceMetadata localResourceMetadata = new LSMInvertedIndexLocalResourceMetadata(invListsTypeTraits,
+ IResourceFactory localResourceMetadata = new LSMInvertedIndexLocalResourceMetadataFactory(invListsTypeTraits,
primaryComparatorFactories, tokenTypeTraits, tokenComparatorFactories, tokenizerFactory, isPartitioned,
dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties, filterTypeTraits,
filterCmpFactories, invertedIndexFields, secondaryFilterFields, secondaryFilterFieldsForNonBulkLoadOps,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
index f1b43f08..8a9549b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -21,7 +21,6 @@
import java.util.List;
import org.apache.asterix.app.external.ExternalIndexingOperations;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -30,6 +29,7 @@
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
+import org.apache.asterix.common.transactions.IResourceFactory;
import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
@@ -44,8 +44,8 @@
import org.apache.asterix.om.util.NonTaggedFormatUtil;
import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
-import org.apache.asterix.transaction.management.resource.ExternalRTreeLocalResourceMetadata;
-import org.apache.asterix.transaction.management.resource.LSMRTreeLocalResourceMetadata;
+import org.apache.asterix.transaction.management.resource.ExternalRTreeLocalResourceMetadataFactory;
+import org.apache.asterix.transaction.management.resource.LSMRTreeLocalResourceMetadataFactory;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -107,7 +107,7 @@
IBinaryComparatorFactory[] btreeCompFactories = getComparatorFactoriesForDeletedKeyBTree();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
- ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(secondaryTypeTraits,
+ IResourceFactory localResourceMetadata = new LSMRTreeLocalResourceMetadataFactory(secondaryTypeTraits,
secondaryComparatorFactories, btreeCompFactories, valueProviderFactories, RTreePolicyType.RTREE,
MetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties, filterTypeTraits,
@@ -124,7 +124,7 @@
} else {
// External dataset
// Prepare a LocalResourceMetadata which will be stored in NC's local resource repository
- ILocalResourceMetadata localResourceMetadata = new ExternalRTreeLocalResourceMetadata(secondaryTypeTraits,
+ IResourceFactory localResourceMetadata = new ExternalRTreeLocalResourceMetadataFactory(secondaryTypeTraits,
secondaryComparatorFactories, ExternalIndexingOperations.getBuddyBtreeComparatorFactories(),
valueProviderFactories, RTreePolicyType.RTREE,
MetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
index d4a15eb..5d87b00 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
@@ -48,7 +48,7 @@
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONTokener;
@@ -68,7 +68,7 @@
ConnectorAPIServlet servlet = spy(new ConnectorAPIServlet());
ServletConfig mockServletConfig = mock(ServletConfig.class);
servlet.init(mockServletConfig);
- Map<String, NodeControllerInfo> nodeMap = new HashMap<String, NodeControllerInfo>();
+ Map<String, NodeControllerInfo> nodeMap = new HashMap<>();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
PrintWriter outputWriter = new PrintWriter(outputStream);
@@ -123,9 +123,9 @@
ConnectorAPIServlet servlet = new ConnectorAPIServlet();
JSONObject actualResponse = new JSONObject();
FileSplit[] splits = new FileSplit[2];
- splits[0] = new FileSplit("asterix_nc1", "foo1");
- splits[1] = new FileSplit("asterix_nc2", "foo2");
- Map<String, NodeControllerInfo> nodeMap = new HashMap<String, NodeControllerInfo>();
+ splits[0] = new FileSplit("asterix_nc1", "foo1", true);
+ splits[1] = new FileSplit("asterix_nc2", "foo2", true);
+ Map<String, NodeControllerInfo> nodeMap = new HashMap<>();
NodeControllerInfo mockInfo1 = mock(NodeControllerInfo.class);
NodeControllerInfo mockInfo2 = mock(NodeControllerInfo.class);
@@ -142,8 +142,9 @@
nodeMap.put("asterix_nc1", mockInfo1);
nodeMap.put("asterix_nc2", mockInfo2);
PA.invokeMethod(servlet,
- "formResponseObject(org.json.JSONObject, org.apache.hyracks.dataflow.std.file.FileSplit[], "
- + "org.apache.asterix.om.types.ARecordType, java.lang.String, boolean, java.util.Map)",
+ "formResponseObject(" + JSONObject.class.getName() + ", " + FileSplit.class.getName() + "[], "
+ + ARecordType.class.getName() + ", " + String.class.getName() + ", boolean, " + Map.class
+ .getName() + ")",
actualResponse, splits, recordType, primaryKey, true, nodeMap);
// Constructs expected response.
@@ -154,12 +155,12 @@
JSONArray splitsArray = new JSONArray();
JSONObject element1 = new JSONObject();
element1.put("ip", "127.0.0.1");
- element1.put("path", splits[0].getLocalFile().getFile().getAbsolutePath());
- element1.put("ioDeviceId", 0);
+ element1.put("path", splits[0].getPath());
+ element1.put("relative", true);
JSONObject element2 = new JSONObject();
element2.put("ip", "127.0.0.2");
- element2.put("path", splits[1].getLocalFile().getFile().getAbsolutePath());
- element2.put("ioDeviceId", 0);
+ element2.put("path", splits[1].getPath());
+ element2.put("relative", true);
splitsArray.put(element1);
splitsArray.put(element2);
expectedResponse.put("splits", splitsArray);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index df8d22e..6ab4b80 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -25,7 +25,6 @@
import org.apache.asterix.algebra.operators.physical.CommitRuntime;
import org.apache.asterix.app.external.TestLibrarian;
import org.apache.asterix.app.nc.AsterixNCAppRuntimeContext;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixTransactionProperties;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.DatasetLifecycleManager;
@@ -34,6 +33,7 @@
import org.apache.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
+import org.apache.asterix.common.transactions.IResourceFactory;
import org.apache.asterix.common.transactions.ITransactionManager;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
@@ -47,7 +47,7 @@
import org.apache.asterix.test.runtime.ExecutionTestUtil;
import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory;
import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
-import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
+import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadataFactory;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import org.apache.asterix.transaction.management.service.logging.LogReader;
import org.apache.asterix.transaction.management.service.transaction.TransactionSubsystem;
@@ -65,12 +65,12 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.util.TaskUtils;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorNodePushable;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
@@ -156,7 +156,7 @@
public AsterixLSMInsertDeleteOperatorNodePushable getInsertPipeline(IHyracksTaskContext ctx, Dataset dataset,
IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields)
- throws AlgebricksException {
+ throws AlgebricksException, HyracksDataException {
PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType,
mergePolicyFactory, mergePolicyProperties, filterFields);
IndexOperation op = IndexOperation.INSERT;
@@ -251,7 +251,7 @@
public ConstantFileSplitProvider getFileSplitProvider(Dataset dataset) {
FileSplit fileSplit = new FileSplit(ExecutionTestUtil.integrationUtil.ncs[0].getId(),
- dataset.getDataverseName() + File.separator + dataset.getDatasetName());
+ dataset.getDataverseName() + File.separator + dataset.getDatasetName(), true);
return new ConstantFileSplitProvider(new FileSplit[] { fileSplit });
}
@@ -260,7 +260,7 @@
int[] primaryIndexBloomFilterKeyFields, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields) {
- ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(primaryIndexTypeTraits,
+ IResourceFactory localResourceMetadata = new LSMBTreeLocalResourceMetadataFactory(primaryIndexTypeTraits,
primaryIndexComparatorFactories, primaryIndexBloomFilterKeyFields, true, dataset.getDatasetId(),
mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories, btreeFields,
filterFields);
@@ -271,7 +271,7 @@
public LSMBTreeDataflowHelper getPrimaryIndexDataflowHelper(IHyracksTaskContext ctx,
PrimaryIndexInfo primaryIndexInfo, TreeIndexCreateOperatorDescriptor indexOpDesc)
- throws AlgebricksException {
+ throws AlgebricksException, HyracksDataException {
LSMBTreeDataflowHelperFactory dataflowHelperFactory = new LSMBTreeDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(primaryIndexInfo.dataset.getDatasetId()),
primaryIndexInfo.mergePolicyFactory, primaryIndexInfo.mergePolicyProperties,
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index b966bcd..0a14100 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -245,6 +245,10 @@
<artifactId>hyracks-api</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-test-framework</artifactId>
<version>${project.version}</version>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java
index b714902..b330271 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java
@@ -18,28 +18,33 @@
*/
package org.apache.asterix.common.context;
-import java.io.File;
import java.util.HashMap;
import java.util.Map;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.common.file.IFileMapManager;
public class AsterixFileMapManager implements IFileMapManager {
private static final long serialVersionUID = 1L;
- private Map<Integer, String> id2nameMap = new HashMap<Integer, String>();
- private Map<String, Integer> name2IdMap = new HashMap<String, Integer>();
+ private final transient IOManager ioManager;
+ private final Map<Integer, String> id2nameMap = new HashMap<>();
+ private final Map<String, Integer> name2IdMap = new HashMap<>();
private int idCounter = 0;
+ public AsterixFileMapManager(IOManager ioManager) {
+ this.ioManager = ioManager;
+ }
+
@Override
public FileReference lookupFileName(int fileId) throws HyracksDataException {
String fName = id2nameMap.get(fileId);
if (fName == null) {
throw new HyracksDataException("No mapping found for id: " + fileId);
}
- return new FileReference(new File(fName));
+ return ioManager.getFileRef(fName, false);
}
@Override
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
index 12faad2..685d12d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
@@ -22,6 +22,11 @@
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
@@ -36,11 +41,23 @@
}
@Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx, IFileSplitProvider fileSplitProvider) {
+ public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx,
+ IFileSplitProvider fileSplitProvider) throws HyracksDataException {
final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
- final int ioDeviceNum = fileSplitProvider.getFileSplits()[partition].getIODeviceId();
+ IIOManager ioManager = ctx.getIOManager();
+ FileSplit fileSplit = fileSplitProvider.getFileSplits()[partition];
+ FileReference fileRef = ioManager.getFileRef(fileSplit.getPath(), fileSplit.isManaged());
+ IODeviceHandle device = fileRef.getDeviceHandle();
+ List<IODeviceHandle> devices = ioManager.getIODevices();
+ int deviceId = 0;
+ for (int i = 0; i < devices.size(); i++) {
+ IODeviceHandle next = devices.get(i);
+ if (next == device) {
+ deviceId = i;
+ }
+ }
return ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject())
- .getDatasetLifecycleManager().getVirtualBufferCaches(datasetID, ioDeviceNum);
+ .getDatasetLifecycleManager().getVirtualBufferCaches(datasetID, deviceId);
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index 698dfb4..a0fe996 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -27,12 +27,12 @@
import java.util.concurrent.ConcurrentHashMap;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
import org.apache.asterix.common.transactions.ILogManager;
import org.apache.asterix.common.transactions.LogRecord;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.asterix.common.utils.TransactionUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
@@ -101,19 +101,19 @@
}
public int getDIDfromResourcePath(String resourcePath) throws HyracksDataException {
- LocalResource lr = resourceRepository.getResourceByPath(resourcePath);
+ LocalResource lr = resourceRepository.get(resourcePath);
if (lr == null) {
return -1;
}
- return ((ILocalResourceMetadata) lr.getResourceObject()).getDatasetID();
+ return ((Resource) lr.getResource()).datasetId();
}
public long getResourceIDfromResourcePath(String resourcePath) throws HyracksDataException {
- LocalResource lr = resourceRepository.getResourceByPath(resourcePath);
+ LocalResource lr = resourceRepository.get(resourcePath);
if (lr == null) {
return -1;
}
- return lr.getResourceId();
+ return lr.getId();
}
@Override
@@ -166,34 +166,34 @@
@Override
public synchronized void open(String resourcePath) throws HyracksDataException {
- validateDatasetLifecycleManagerState();
- int did = getDIDfromResourcePath(resourcePath);
- long resourceID = getResourceIDfromResourcePath(resourcePath);
+ validateDatasetLifecycleManagerState();
+ int did = getDIDfromResourcePath(resourcePath);
+ long resourceID = getResourceIDfromResourcePath(resourcePath);
- DatasetResource dsr = datasets.get(did);
- DatasetInfo dsInfo = dsr.getDatasetInfo();
- if (dsInfo == null || !dsInfo.isRegistered()) {
- throw new HyracksDataException(
- "Failed to open index with resource ID " + resourceID + " since it does not exist.");
- }
-
- IndexInfo iInfo = dsInfo.getIndexes().get(resourceID);
- if (iInfo == null) {
- throw new HyracksDataException(
- "Failed to open index with resource ID " + resourceID + " since it does not exist.");
- }
-
- dsr.open(true);
- dsr.touch();
-
- if (!iInfo.isOpen()) {
- ILSMOperationTracker opTracker = iInfo.getIndex().getOperationTracker();
- synchronized (opTracker) {
- iInfo.getIndex().activate();
+ DatasetResource dsr = datasets.get(did);
+ DatasetInfo dsInfo = dsr.getDatasetInfo();
+ if (dsInfo == null || !dsInfo.isRegistered()) {
+ throw new HyracksDataException(
+ "Failed to open index with resource ID " + resourceID + " since it does not exist.");
}
- iInfo.setOpen(true);
- }
- iInfo.touch();
+
+ IndexInfo iInfo = dsInfo.getIndexes().get(resourceID);
+ if (iInfo == null) {
+ throw new HyracksDataException(
+ "Failed to open index with resource ID " + resourceID + " since it does not exist.");
+ }
+
+ dsr.open(true);
+ dsr.touch();
+
+ if (!iInfo.isOpen()) {
+ ILSMOperationTracker opTracker = iInfo.getIndex().getOperationTracker();
+ synchronized (opTracker) {
+ iInfo.getIndex().activate();
+ }
+ iInfo.setOpen(true);
+ }
+ iInfo.touch();
}
private boolean evictCandidateDataset() throws HyracksDataException {
@@ -282,7 +282,7 @@
@Override
public synchronized List<IIndex> getOpenResources() {
List<IndexInfo> openIndexesInfo = getOpenIndexesInfo();
- List<IIndex> openIndexes = new ArrayList<IIndex>();
+ List<IIndex> openIndexes = new ArrayList<>();
for (IndexInfo iInfo : openIndexesInfo) {
openIndexes.add(iInfo.getIndex());
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java
index 01f2e2b..403d3cb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java
@@ -83,10 +83,7 @@
public IIndex getIndex(long resourceID) {
IndexInfo iInfo = getIndexInfo(resourceID);
- if (iInfo == null) {
- return null;
- }
- return iInfo.getIndex();
+ return (iInfo == null) ? null : iInfo.getIndex();
}
public void register(long resourceID, IIndex index) throws HyracksDataException {
@@ -102,6 +99,9 @@
if (datasetInfo.getIndexes().containsKey(resourceID)) {
throw new HyracksDataException("Index with resource ID " + resourceID + " already exists.");
}
+ if (index == null) {
+ throw new HyracksDataException("Attempt to register a null index");
+ }
datasetInfo.getIndexes().put(resourceID,
new IndexInfo((ILSMIndex) index, datasetInfo.getDatasetID(), resourceID));
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
index fb12808..9ebb5f2 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
@@ -66,7 +66,7 @@
public AsterixLSMInsertDeleteOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, IndexOperation op,
- boolean isPrimary) {
+ boolean isPrimary) throws HyracksDataException {
super(opDesc, ctx, partition, fieldPermutation, recordDescProvider, op);
this.isPrimary = isPrimary;
}
@@ -86,12 +86,11 @@
}
writer.open();
modCallback = opDesc.getModificationOpCallbackFactory().createModificationOperationCallback(
- indexHelper.getResourcePath(), indexHelper.getResourceID(), indexHelper.getResourcePartition(),
- lsmIndex, ctx, this);
+ indexHelper.getResource(), ctx, this);
indexAccessor = lsmIndex.createAccessor(modCallback, NoOpOperationCallback.INSTANCE);
ITupleFilterFactory tupleFilterFactory = opDesc.getTupleFilterFactory();
if (tupleFilterFactory != null) {
- tupleFilter = tupleFilterFactory.createTupleFilter(indexHelper.getTaskContext());
+ tupleFilter = tupleFilterFactory.createTupleFilter(ctx);
frameTuple = new FrameTupleReference();
}
IAsterixAppRuntimeContext runtimeCtx =
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInvertedIndexInsertDeleteOperatorDescriptor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInvertedIndexInsertDeleteOperatorDescriptor.java
index 0d7d1d7..6255172 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInvertedIndexInsertDeleteOperatorDescriptor.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInvertedIndexInsertDeleteOperatorDescriptor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -56,7 +57,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new AsterixLSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation,
recordDescProvider, op, false);
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IResourceFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IResourceFactory.java
new file mode 100644
index 0000000..d569855
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IResourceFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.transactions;
+
+import java.io.Serializable;
+
+@FunctionalInterface
+public interface IResourceFactory extends Serializable {
+ /**
+ * Create a serializable resource for the task partition
+ * @param partition
+ * Hyracks task partition
+ * @return the serializable resource
+ */
+ Resource resource(int partition);
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java
new file mode 100644
index 0000000..d649782
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.transactions;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.common.file.LocalResource;
+
+/**
+ * The base resource that will be written to disk. it will go in the serializable resource
+ * member in {@link LocalResource}
+ */
+public abstract class Resource implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final int datasetId;
+ private final int partition;
+ protected final ITypeTraits[] filterTypeTraits;
+ protected final IBinaryComparatorFactory[] filterCmpFactories;
+ protected final int[] filterFields;
+
+ public Resource(int datasetId, int partition, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
+ this.datasetId = datasetId;
+ this.partition = partition;
+ this.filterTypeTraits = filterTypeTraits;
+ this.filterCmpFactories = filterCmpFactories;
+ this.filterFields = filterFields;
+ }
+
+ public int partition() {
+ return partition;
+ }
+
+ public int datasetId() {
+ return datasetId;
+ }
+
+ public abstract ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider,
+ LocalResource resource) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/AbstractLSMLocalResourceMetadata.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java
similarity index 75%
rename from asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/AbstractLSMLocalResourceMetadata.java
rename to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java
index 3367d2e..f7f7a71 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/AbstractLSMLocalResourceMetadata.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java
@@ -16,30 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.transaction.management.resource;
+package org.apache.asterix.common.transactions;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-public abstract class AbstractLSMLocalResourceMetadata implements ILocalResourceMetadata {
+public abstract class ResourceFactory implements IResourceFactory {
private static final long serialVersionUID = 1L;
-
- protected final int datasetID;
+ protected final int datasetId;
protected final ITypeTraits[] filterTypeTraits;
protected final IBinaryComparatorFactory[] filterCmpFactories;
protected final int[] filterFields;
- public AbstractLSMLocalResourceMetadata(int datasetID, ITypeTraits[] filterTypeTraits,
+ public ResourceFactory(int datasetId, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
- this.datasetID = datasetID;
+ this.datasetId = datasetId;
this.filterTypeTraits = filterTypeTraits;
this.filterCmpFactories = filterCmpFactories;
this.filterFields = filterFields;
}
-
- public int getDatasetID() {
- return datasetID;
- }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
index 615e8af..5816450 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -25,9 +25,8 @@
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -47,12 +46,11 @@
loc[p] = splits[p].getNodeName();
}
AlgebricksPartitionConstraint pc = new AlgebricksAbsolutePartitionConstraint(loc);
- return new Pair<IFileSplitProvider, AlgebricksPartitionConstraint>(splitProvider, pc);
+ return new Pair<>(splitProvider, pc);
}
- public static FileSplit getFileSplitForClusterPartition(ClusterPartition partition, File relativeFile) {
- return new FileSplit(partition.getActiveNodeId(), new FileReference(relativeFile), partition.getIODeviceNum(),
- partition.getPartitionId());
+ public static FileSplit getFileSplitForClusterPartition(ClusterPartition partition, String relativePath) {
+ return new FileSplit(partition.getActiveNodeId(), relativePath, true);
}
public static String prepareStoragePartitionPath(String storageDirName, int partitonId) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index 275825b..4853129 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -47,7 +47,7 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class GenericAdapterFactory implements IIndexingAdapterFactory, IAdapterFactory {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
index 667bae7..42e11b1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
@@ -92,7 +92,7 @@
// create the accessor and the cursor using the passed version
ISearchOperationCallback searchCallback = opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(indexDataflowHelper.getResourceID(), ctx, null);
+ .createSearchOperationCallback(indexDataflowHelper.getResource().getId(), ctx, null);
fileIndexAccessor = index.createAccessor(searchCallback, indexDataflowHelper.getVersion());
fileIndexSearchCursor = fileIndexAccessor.createSearchCursor(false);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
index ae012f3..7aeb0bc 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.external.input.stream.factory;
-import java.io.File;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Map;
@@ -40,8 +39,7 @@
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class LocalFSInputStreamFactory implements IInputStreamFactory {
@@ -66,7 +64,7 @@
ArrayList<Path> inputResources = new ArrayList<>();
for (int i = 0; i < inputFileSplits.length; i++) {
if (inputFileSplits[i].getNodeName().equals(nodeName)) {
- inputResources.add(inputFileSplits[i].getLocalFile().getFile().toPath());
+ inputResources.add(inputFileSplits[i].getFile(ctx.getIOManager()).toPath());
}
}
watcher = new FileSystemWatcher(inputResources, expression, isFeed);
@@ -115,7 +113,7 @@
}
nodeName = resolver.resolveNode(trimmedValue.split(":")[0]);
nodeLocalPath = trimmedValue.split("://")[1];
- FileSplit fileSplit = new FileSplit(nodeName, new FileReference(new File(nodeLocalPath)));
+ FileSplit fileSplit = new FileSplit(nodeName, nodeLocalPath, false);
inputFileSplits[count++] = fileSplit;
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java
index 9b37722..078e4f8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java
@@ -26,6 +26,8 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
@@ -78,8 +80,10 @@
@Override
public void initialize() throws HyracksDataException {
try {
+ FileSplit fileSplit = fileIndexInfo.getFileSplitProvider().getFileSplits()[partition];
+ FileReference fileRef = fileSplit.getFileRef(ctx.getIOManager());
// only in partition of device id = 0, we perform the operation on the files index
- if (fileIndexInfo.getFileSplitProvider().getFileSplits()[partition].getIODeviceId() == 0) {
+ if (fileRef.getDeviceHandle() == ctx.getIOManager().getIODevices().get(0)) {
performOpOnIndex(filesIndexDataflowHelperFactory, ctx, fileIndexInfo, partition);
}
// perform operation on btrees
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java
index c1903e0..8202316 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
@@ -52,7 +53,7 @@
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new ExternalBTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorNodePushable.java
index 9b83a12..69a2020 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorNodePushable.java
@@ -38,7 +38,7 @@
public ExternalBTreeSearchOperatorNodePushable(ExternalBTreeSearchOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition, IRecordDescriptorProvider recordDescProvider, int[] lowKeyFields,
- int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive) {
+ int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive) throws HyracksDataException {
super(opDesc, ctx, partition, recordDescProvider, lowKeyFields, highKeyFields, lowKeyInclusive,
highKeyInclusive, null, null);
}
@@ -73,7 +73,7 @@
dos = tb.getDataOutput();
appender = new FrameTupleAppender(new VSizeFrame(ctx));
ISearchOperationCallback searchCallback = opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(indexHelper.getResourceID(), ctx, null);
+ .createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
// The next line is the reason we override this method
indexAccessor = externalIndex.createAccessor(searchCallback, dataFlowHelper.getTargetVersion());
cursor = createCursor();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java
index 5255257..8321074 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java
@@ -49,7 +49,7 @@
protected void performOpOnIndex(IIndexDataflowHelperFactory indexDataflowHelperFactory, IHyracksTaskContext ctx,
IndexInfoOperatorDescriptor fileIndexInfo, int partition) throws Exception {
FileReference file = IndexFileNameUtil.getIndexAbsoluteFileRef(fileIndexInfo, partition, ctx.getIOManager());
- AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(file);
+ AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(ctx.getIOManager(), file);
fileManager.deleteTransactionFiles();
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java
index 82ca715..0312eac 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java
@@ -49,7 +49,7 @@
protected void performOpOnIndex(IIndexDataflowHelperFactory indexDataflowHelperFactory, IHyracksTaskContext ctx,
IndexInfoOperatorDescriptor fileIndexInfo, int partition) throws Exception {
FileReference file = IndexFileNameUtil.getIndexAbsoluteFileRef(fileIndexInfo, partition, ctx.getIOManager());
- AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(file);
+ AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(ctx.getIOManager(), file);
fileManager.recoverTransaction();
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
index 5748a65..c95a4a7 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
@@ -45,7 +45,7 @@
public ExternalIndexBulkModifyOperatorNodePushable(ExternalIndexBulkModifyOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition, int[] fieldPermutation, float fillFactor, long numElementsHint,
- IRecordDescriptorProvider recordDescProvider, int[] deletedFiles) {
+ IRecordDescriptorProvider recordDescProvider, int[] deletedFiles) throws HyracksDataException {
super(opDesc, ctx, partition, fieldPermutation, fillFactor, false, numElementsHint, false, recordDescProvider);
this.deletedFiles = deletedFiles;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java
index e217365..c0907ee 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -50,7 +51,7 @@
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new ExternalRTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, keyFields);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorNodePushable.java
index cda2985..97a38747 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorNodePushable.java
@@ -38,7 +38,7 @@
public class ExternalRTreeSearchOperatorNodePushable extends RTreeSearchOperatorNodePushable {
public ExternalRTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, IRecordDescriptorProvider recordDescProvider, int[] keyFields) {
+ int partition, IRecordDescriptorProvider recordDescProvider, int[] keyFields) throws HyracksDataException {
super(opDesc, ctx, partition, recordDescProvider, keyFields, null, null);
}
@@ -72,7 +72,7 @@
dos = tb.getDataOutput();
appender = new FrameTupleAppender(new VSizeFrame(ctx));
ISearchOperationCallback searchCallback = opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(indexHelper.getResourceID(), ctx, null);
+ .createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
// The next line is the reason we override this method
indexAccessor = rTreeIndex.createAccessor(searchCallback, rTreeDataflowHelper.getTargetVersion());
cursor = createCursor();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index 366534f..303f76d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -37,9 +37,9 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.util.IntSerDeUtils;
public class FeedUtils {
@@ -82,7 +82,7 @@
// Note: feed adapter instances in a single node share the feed logger
// format: 'storage dir name'/partition_#/dataverse/feed/node
File f = new File(storagePartitionPath + File.separator + relPathFile + File.separator + nodeName);
- return StoragePathUtil.getFileSplitForClusterPartition(partition, f);
+ return StoragePathUtil.getFileSplitForClusterPartition(partition, f.getPath());
}
public static FileSplit[] splitsForAdapter(String dataverseName, String feedName,
@@ -100,20 +100,19 @@
}
public static FileReference getAbsoluteFileRef(String relativePath, int ioDeviceId, IIOManager ioManager) {
- return ioManager.getAbsoluteFileRef(ioDeviceId, relativePath);
+ return ioManager.getFileRef(ioDeviceId, relativePath);
}
public static FeedLogManager getFeedLogManager(IHyracksTaskContext ctx, int partition,
FileSplit[] feedLogFileSplits) throws HyracksDataException {
return new FeedLogManager(
- FeedUtils.getAbsoluteFileRef(feedLogFileSplits[partition].getLocalFile().getFile().getPath(),
- feedLogFileSplits[partition].getIODeviceId(), ctx.getIOManager()).getFile());
+ FeedUtils.getAbsoluteFileRef(feedLogFileSplits[partition].getPath(), 0, ctx.getIOManager()).getFile());
}
public static FeedLogManager getFeedLogManager(IHyracksTaskContext ctx, FileSplit feedLogFileSplit)
throws HyracksDataException {
- return new FeedLogManager(FeedUtils.getAbsoluteFileRef(feedLogFileSplit.getLocalFile().getFile().getPath(),
- feedLogFileSplit.getIODeviceId(), ctx.getIOManager()).getFile());
+ return new FeedLogManager(FeedUtils.getAbsoluteFileRef(feedLogFileSplit.getPath(),
+ 0, ctx.getIOManager()).getFile());
}
public static void processFeedMessage(ByteBuffer input, VSizeFrame message, FrameTupleAccessor fta)
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 6de796e..7d0d91c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -406,7 +406,7 @@
private void insertTupleIntoIndex(JobId jobId, IMetadataIndex metadataIndex, ITupleReference tuple)
throws ACIDException, HyracksDataException, IndexException {
long resourceID = metadataIndex.getResourceID();
- String resourceName = metadataIndex.getFile().toString();
+ String resourceName = metadataIndex.getFile().getRelativePath();
ILSMIndex lsmIndex = (ILSMIndex) datasetLifecycleManager.get(resourceName);
try {
datasetLifecycleManager.open(resourceName);
@@ -673,7 +673,7 @@
private void deleteTupleFromIndex(JobId jobId, IMetadataIndex metadataIndex, ITupleReference tuple)
throws ACIDException, HyracksDataException, IndexException {
long resourceID = metadataIndex.getResourceID();
- String resourceName = metadataIndex.getFile().toString();
+ String resourceName = metadataIndex.getFile().getRelativePath();
ILSMIndex lsmIndex = (ILSMIndex) datasetLifecycleManager.get(resourceName);
try {
datasetLifecycleManager.open(resourceName);
@@ -1148,7 +1148,7 @@
if (index.getFile() == null) {
throw new MetadataException("No file for Index " + index.getDataverseName() + "." + index.getIndexName());
}
- String resourceName = index.getFile().toString();
+ String resourceName = index.getFile().getRelativePath();
IIndex indexInstance = datasetLifecycleManager.get(resourceName);
datasetLifecycleManager.open(resourceName);
IIndexAccessor indexAccessor = indexInstance.createAccessor(NoOpOperationCallback.INSTANCE,
@@ -1186,7 +1186,7 @@
public void initializeDatasetIdFactory(JobId jobId) throws MetadataException, RemoteException {
int mostRecentDatasetId = MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID;
try {
- String resourceName = MetadataPrimaryIndexes.DATASET_DATASET.getFile().toString();
+ String resourceName = MetadataPrimaryIndexes.DATASET_DATASET.getFile().getRelativePath();
IIndex indexInstance = datasetLifecycleManager.get(resourceName);
datasetLifecycleManager.open(resourceName);
try {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 0990998..17ef709 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -28,7 +28,6 @@
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.config.AsterixMetadataProperties;
import org.apache.asterix.common.config.ClusterProperties;
@@ -39,6 +38,7 @@
import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.adapter.factory.GenericAdapterFactory;
import org.apache.asterix.external.api.IAdapterFactory;
@@ -341,7 +341,7 @@
String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath(
ClusterProperties.INSTANCE.getStorageDirectoryName(), metadataPartition.getPartitionId());
String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath();
- FileReference file = ioManager.getAbsoluteFileRef(metadataDeviceId, resourceName);
+ FileReference file = ioManager.getFileRef(metadataDeviceId, resourceName);
// this should not be done this way. dataset lifecycle manager shouldn't return virtual buffer caches for
// a dataset that was not yet created
@@ -356,9 +356,9 @@
index.isPrimaryIndex() ? runtimeContext.getLSMBTreeOperationTracker(index.getDatasetId().getId())
: new BaseOperationTracker(index.getDatasetId().getId(),
dataLifecycleManager.getDatasetInfo(index.getDatasetId().getId()));
- final String absolutePath = file.getFile().getPath();
if (isNewUniverse()) {
- lsmBtree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, bufferCache, fileMapProvider, typeTraits,
+ lsmBtree = LSMBTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, fileMapProvider,
+ typeTraits,
comparatorFactories, bloomFilterKeyFields, runtimeContext.getBloomFilterFalsePositiveRate(),
runtimeContext.getMetadataMergePolicyFactory()
.createMergePolicy(GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, dataLifecycleManager),
@@ -367,28 +367,32 @@
null, null, null, null, true);
lsmBtree.create();
resourceID = index.getResourceID();
- ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
+ Resource localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(), index.getDatasetId().getId(),
+ metadataPartition.getPartitionId(),
runtimeContext.getMetadataMergePolicyFactory(), GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES,
null, null, null, null);
ILocalResourceFactoryProvider localResourceFactoryProvider =
- new PersistentLocalResourceFactoryProvider(localResourceMetadata, LocalResource.LSMBTreeResource);
+ new PersistentLocalResourceFactoryProvider(partition -> localResourceMetadata,
+ LocalResource.LSMBTreeResource);
ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory();
localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, resourceName,
- metadataPartition.getPartitionId(), LIFOMetaDataFrame.VERSION, absolutePath));
- dataLifecycleManager.register(absolutePath, lsmBtree);
+ LIFOMetaDataFrame.VERSION, metadataPartition.getPartitionId()));
+ dataLifecycleManager.register(file.getRelativePath(), lsmBtree);
} else {
- final LocalResource resource = localResourceRepository.getResourceByPath(absolutePath);
+ final LocalResource resource = localResourceRepository.get(file.getRelativePath());
if (resource == null) {
throw new HyracksDataException("Could not find required metadata indexes. Please delete "
+ propertiesProvider.getMetadataProperties().getTransactionLogDirs()
.get(runtimeContext.getTransactionSubsystem().getId())
+ " to intialize as a new instance. (WARNING: all data will be lost.)");
}
- resourceID = resource.getResourceId();
- lsmBtree = (LSMBTree) dataLifecycleManager.get(absolutePath);
+ resourceID = resource.getId();
+ assert (index.getResourceID() == resource.getId());
+ lsmBtree = (LSMBTree) dataLifecycleManager.get(file.getRelativePath());
if (lsmBtree == null) {
- lsmBtree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, bufferCache, fileMapProvider,
+ lsmBtree = LSMBTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache,
+ fileMapProvider,
typeTraits, comparatorFactories, bloomFilterKeyFields,
runtimeContext.getBloomFilterFalsePositiveRate(),
runtimeContext.getMetadataMergePolicyFactory().createMergePolicy(
@@ -396,7 +400,7 @@
opTracker, runtimeContext.getLSMIOScheduler(),
LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), index.isPrimaryIndex(),
null, null, null, null, true);
- dataLifecycleManager.register(absolutePath, lsmBtree);
+ dataLifecycleManager.register(file.getRelativePath(), lsmBtree);
}
}
index.setResourceID(resourceID);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FileSplitSinkId.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FileSplitSinkId.java
index d4fe646..ba40430 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FileSplitSinkId.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FileSplitSinkId.java
@@ -19,7 +19,7 @@
package org.apache.asterix.metadata.declared;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class FileSplitSinkId {
@@ -31,7 +31,7 @@
@Override
public String toString() {
- return fileSplit.getNodeName() + ":" + fileSplit.getLocalFile();
+ return fileSplit.getNodeName() + ":" + fileSplit.getPath();
}
public FileSplit getFileSplit() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 7258be8..3ec10f0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -138,11 +138,11 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.ShortPointable;
import org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
@@ -723,7 +723,7 @@
FileSplitDataSink fsds = (FileSplitDataSink) sink;
FileSplitSinkId fssi = fsds.getId();
FileSplit fs = fssi.getFileSplit();
- File outFile = fs.getLocalFile().getFile();
+ File outFile = new File(fs.getPath());
String nodeId = fs.getNodeName();
SinkWriterRuntimeFactory runtime = new SinkWriterRuntimeFactory(printColumns, printerFactories, outFile,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
index 21e2150..24d185f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
@@ -33,7 +33,7 @@
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
public class SplitsAndConstraintsUtil {
@@ -48,13 +48,9 @@
ClusterPartition[] clusterPartition = ClusterStateManager.INSTANCE.getClusterPartitons();
String storageDirName = ClusterProperties.INSTANCE.getStorageDirectoryName();
for (int j = 0; j < clusterPartition.length; j++) {
- int nodePartitions =
- ClusterStateManager.INSTANCE.getNodePartitionsCount(clusterPartition[j].getNodeId());
- for (int i = 0; i < nodePartitions; i++) {
File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
- clusterPartition[i].getPartitionId()) + File.separator + relPathFile);
- splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition[j], f));
- }
+ clusterPartition[j].getPartitionId()) + File.separator + relPathFile);
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition[j], f.getPath()));
}
return splits.toArray(new FileSplit[] {});
}
@@ -87,7 +83,7 @@
nodePartitions[k].getPartitionId())
+ (temp ? (File.separator + StoragePathUtil.TEMP_DATASETS_STORAGE_FOLDER) : "")
+ File.separator + relPathFile);
- splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[k], f));
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[k], f.getPath()));
}
}
return splits.toArray(new FileSplit[] {});
@@ -118,12 +114,14 @@
// Only the first partition when create
File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
nodePartitions[firstPartition].getPartitionId()) + File.separator + relPathFile);
- splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[firstPartition], f));
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[firstPartition], f
+ .getPath()));
} else {
for (int k = 0; k < nodePartitions.length; k++) {
File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
nodePartitions[firstPartition].getPartitionId()) + File.separator + relPathFile);
- splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[firstPartition], f));
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[firstPartition], f
+ .getPath()));
}
}
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/ReplicaResourcesManager.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/ReplicaResourcesManager.java
index 6fc8fd5..d0a5ac6 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/ReplicaResourcesManager.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/ReplicaResourcesManager.java
@@ -166,7 +166,7 @@
File localResource = new File(
indexFolder + File.separator + PersistentLocalResourceRepository.METADATA_FILE_NAME);
LocalResource resource = PersistentLocalResourceRepository.readLocalResource(localResource);
- laggingReplicaIndexes.put(resource.getResourceId(), indexFolder.getAbsolutePath());
+ laggingReplicaIndexes.put(resource.getId(), indexFolder.getAbsolutePath());
}
}
} catch (HyracksDataException e) {
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReportMaxResourceIdMessage.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReportMaxResourceIdMessage.java
index c1319e0..b4bbe01 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReportMaxResourceIdMessage.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReportMaxResourceIdMessage.java
@@ -58,7 +58,7 @@
NodeControllerService ncs = cs;
IAsterixAppRuntimeContext appContext =
(IAsterixAppRuntimeContext) ncs.getApplicationContext().getApplicationObject();
- long maxResourceId = Math.max(appContext.getLocalResourceRepository().getMaxResourceID(),
+ long maxResourceId = Math.max(appContext.getLocalResourceRepository().maxId(),
MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID);
ReportMaxResourceIdMessage maxResourceIdMsg = new ReportMaxResourceIdMessage(ncs.getId(), maxResourceId);
try {
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMInvertedIndexUpsertOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMInvertedIndexUpsertOperatorDescriptor.java
index 3db3de2..50c5d0c 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMInvertedIndexUpsertOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMInvertedIndexUpsertOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -58,7 +59,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new AsterixLSMSecondaryUpsertOperatorNodePushable(this, ctx, partition, fieldPermutation,
recordDescProvider, prevFieldPermutation);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
index 536366f..aad7222 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
@@ -86,7 +86,7 @@
public AsterixLSMPrimaryUpsertOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, int numOfPrimaryKeys,
- ARecordType recordType, int filterFieldIndex) {
+ ARecordType recordType, int filterFieldIndex) throws HyracksDataException {
super(opDesc, ctx, partition, fieldPermutation, recordDescProvider, IndexOperation.UPSERT);
this.key = new PermutingFrameTupleReference();
this.numOfPrimaryKeys = numOfPrimaryKeys;
@@ -141,10 +141,9 @@
dos = tb.getDataOutput();
appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
modCallback = opDesc.getModificationOpCallbackFactory().createModificationOperationCallback(
- indexHelper.getResourcePath(), indexHelper.getResourceID(), indexHelper.getResourcePartition(),
- index, ctx, this);
+ indexHelper.getResource(), ctx, this);
searchCallback = (LockThenSearchOperationCallback) opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(indexHelper.getResourceID(), ctx, this);
+ .createSearchOperationCallback(indexHelper.getResource().getId(), ctx, this);
indexAccessor = index.createAccessor(modCallback, searchCallback);
cursor = indexAccessor.createSearchCursor(false);
frameTuple = new FrameTupleReference();
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMSecondaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMSecondaryUpsertOperatorNodePushable.java
index 05b633d..b800c24 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMSecondaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMSecondaryUpsertOperatorNodePushable.java
@@ -56,7 +56,7 @@
public AsterixLSMSecondaryUpsertOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider,
- int[] prevValuePermutation) {
+ int[] prevValuePermutation) throws HyracksDataException {
super(opDesc, ctx, partition, fieldPermutation, recordDescProvider, IndexOperation.UPSERT);
this.prevValueTuple.setFieldPermutation(prevValuePermutation);
this.numberOfFields = prevValuePermutation.length;
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallbackFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallbackFactory.java
index e5f3555..4cbb8cf 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallbackFactory.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallbackFactory.java
@@ -26,14 +26,17 @@
import org.apache.asterix.common.transactions.ITransactionContext;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
+import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.common.file.LocalResource;
/**
* Assumes LSM-BTrees as primary indexes.
@@ -54,24 +57,23 @@
}
@Override
- public IModificationOperationCallback createModificationOperationCallback(String resourcePath, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
- throws HyracksDataException {
-
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException {
ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx);
- IResourceLifecycleManager indexLifeCycleManager =
+ IResourceLifecycleManager<IIndex> indexLifeCycleManager =
txnSubsystem.getAsterixAppRuntimeContextProvider().getDatasetLifecycleManager();
- ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resourcePath);
+ ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath());
if (index == null) {
- throw new HyracksDataException("Index(id:" + resourceId + ") is not registered.");
+ throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered.");
}
try {
ITransactionContext txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(jobId, false);
+ Resource aResource = (Resource) resource.getResource();
IModificationOperationCallback modCallback = new PrimaryIndexModificationOperationCallback(datasetId,
- primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resourceId,
- resourcePartition, resourceType, indexOp, operatorNodePushable, logBeforeImage);
- txnCtx.registerIndexAndCallback(resourceId, index, (AbstractOperationCallback) modCallback, true);
+ primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(),
+ aResource.partition(), resourceType, indexOp, operatorNodePushable, logBeforeImage);
+ txnCtx.registerIndexAndCallback(resource.getId(), index, (AbstractOperationCallback) modCallback, true);
return modCallback;
} catch (ACIDException e) {
throw new HyracksDataException(e);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallbackFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallbackFactory.java
index f9f2b89..3925bba 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallbackFactory.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallbackFactory.java
@@ -26,14 +26,16 @@
import org.apache.asterix.common.transactions.ITransactionContext;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class SecondaryIndexModificationOperationCallbackFactory extends AbstractOperationCallbackFactory
implements IModificationOperationCallbackFactory {
@@ -51,23 +53,23 @@
}
@Override
- public IModificationOperationCallback createModificationOperationCallback(String resourcePath, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
- throws HyracksDataException {
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException {
ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx);
IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getAsterixAppRuntimeContextProvider()
.getDatasetLifecycleManager();
- ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resourcePath);
+ ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath());
if (index == null) {
- throw new HyracksDataException("Index(id:" + resourceId + ") is not registered.");
+ throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered.");
}
try {
ITransactionContext txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(jobId, false);
+ Resource aResource = (Resource) resource.getResource();
IModificationOperationCallback modCallback = new SecondaryIndexModificationOperationCallback(datasetId,
- primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resourceId,
- resourcePartition, resourceType, indexOp, logBeforeImage);
- txnCtx.registerIndexAndCallback(resourceId, index, (AbstractOperationCallback) modCallback, false);
+ primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(),
+ aResource.partition(), resourceType, indexOp, logBeforeImage);
+ txnCtx.registerIndexAndCallback(resource.getId(), index, (AbstractOperationCallback) modCallback, false);
return modCallback;
} catch (ACIDException e) {
throw new HyracksDataException(e);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java
index df7d65d..6e07ea3 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java
@@ -26,14 +26,16 @@
import org.apache.asterix.common.transactions.ITransactionContext;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class TempDatasetPrimaryIndexModificationOperationCallbackFactory extends AbstractOperationCallbackFactory
implements IModificationOperationCallbackFactory {
@@ -49,23 +51,23 @@
}
@Override
- public IModificationOperationCallback createModificationOperationCallback(String resourcePath, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
- throws HyracksDataException {
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException {
ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx);
IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getAsterixAppRuntimeContextProvider()
.getDatasetLifecycleManager();
- ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resourcePath);
+ ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath());
if (index == null) {
- throw new HyracksDataException("Index(id:" + resourceId + ") is not registered.");
+ throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered.");
}
try {
ITransactionContext txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(jobId, false);
+ Resource aResource = (Resource) resource.getResource();
IModificationOperationCallback modCallback = new TempDatasetIndexModificationOperationCallback(datasetId,
- primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resourceId,
- resourcePartition, resourceType, indexOp);
- txnCtx.registerIndexAndCallback(resourceId, index, (AbstractOperationCallback) modCallback, true);
+ primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(),
+ aResource.partition(), resourceType, indexOp);
+ txnCtx.registerIndexAndCallback(resource.getId(), index, (AbstractOperationCallback) modCallback, true);
return modCallback;
} catch (ACIDException e) {
throw new HyracksDataException(e);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java
index 39e916b4..2740994 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java
@@ -26,14 +26,16 @@
import org.apache.asterix.common.transactions.ITransactionContext;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class TempDatasetSecondaryIndexModificationOperationCallbackFactory extends AbstractOperationCallbackFactory
implements IModificationOperationCallbackFactory {
@@ -49,23 +51,23 @@
}
@Override
- public IModificationOperationCallback createModificationOperationCallback(String resourcePath, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
- throws HyracksDataException {
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException {
+ Resource aResource = (Resource) resource.getResource();
ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx);
IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getAsterixAppRuntimeContextProvider()
.getDatasetLifecycleManager();
- ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resourcePath);
+ ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath());
if (index == null) {
- throw new HyracksDataException("Index(id:" + resourceId + ") is not registered.");
+ throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered.");
}
try {
ITransactionContext txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(jobId, false);
IModificationOperationCallback modCallback = new TempDatasetIndexModificationOperationCallback(datasetId,
- primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resourceId,
- resourcePartition, resourceType, indexOp);
- txnCtx.registerIndexAndCallback(resourceId, index, (AbstractOperationCallback) modCallback, false);
+ primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(),
+ aResource.partition(), resourceType, indexOp);
+ txnCtx.registerIndexAndCallback(resource.getId(), index, (AbstractOperationCallback) modCallback, false);
return modCallback;
} catch (ACIDException e) {
throw new HyracksDataException(e);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/UpsertOperationCallbackFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/UpsertOperationCallbackFactory.java
index 85cdc89..9349e93 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/UpsertOperationCallbackFactory.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/UpsertOperationCallbackFactory.java
@@ -25,14 +25,16 @@
import org.apache.asterix.common.transactions.ITransactionContext;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class UpsertOperationCallbackFactory extends AbstractOperationCallbackFactory
implements IModificationOperationCallbackFactory {
@@ -50,24 +52,23 @@
}
@Override
- public IModificationOperationCallback createModificationOperationCallback(String resourceName, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
- throws HyracksDataException {
-
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException {
+ Resource aResource = (Resource) resource.getResource();
ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx);
- IResourceLifecycleManager indexLifeCycleManager =
- txnSubsystem.getAsterixAppRuntimeContextProvider().getDatasetLifecycleManager();
- ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resourceName);
+ IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getAsterixAppRuntimeContextProvider()
+ .getDatasetLifecycleManager();
+ ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath());
if (index == null) {
- throw new HyracksDataException("Index(id:" + resourceId + ") is not registered.");
+ throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered.");
}
try {
ITransactionContext txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(jobId, false);
- IModificationOperationCallback modCallback =
- new UpsertOperationCallback(datasetId, primaryKeyFields, txnCtx, txnSubsystem.getLockManager(),
- txnSubsystem, resourceId, resourcePartition, resourceType, indexOp, logBeforeImage);
- txnCtx.registerIndexAndCallback(resourceId, index, (AbstractOperationCallback) modCallback, true);
+ IModificationOperationCallback modCallback = new UpsertOperationCallback(datasetId, primaryKeyFields,
+ txnCtx, txnSubsystem.getLockManager(),
+ txnSubsystem, resource.getId(), aResource.partition(), resourceType, indexOp, logBeforeImage);
+ txnCtx.registerIndexAndCallback(resource.getId(), index, (AbstractOperationCallback) modCallback, true);
return modCallback;
} catch (ACIDException e) {
throw new HyracksDataException(e);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
index b76e829..3083cfe 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.transaction.management.resource;
-import java.io.File;
import java.util.Map;
import org.apache.asterix.common.context.BaseOperationTracker;
@@ -26,35 +25,39 @@
import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class ExternalBTreeLocalResourceMetadata extends LSMBTreeLocalResourceMetadata {
private static final long serialVersionUID = 1L;
public ExternalBTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields, boolean isPrimary, int datasetID, ILSMMergePolicyFactory mergePolicyFactory,
+ int[] bloomFilterKeyFields, boolean isPrimary, int datasetID, int partition,
+ ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties) {
- super(typeTraits, cmpFactories, bloomFilterKeyFields, isPrimary, datasetID, mergePolicyFactory,
+ super(typeTraits, cmpFactories, bloomFilterKeyFields, isPrimary, datasetID, partition, mergePolicyFactory,
mergePolicyProperties, null, null, null, null);
}
@Override
- public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) {
- FileReference file = new FileReference(new File(filePath));
-
- LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(file, runtimeContextProvider.getBufferCache(),
+ public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider,
+ LocalResource resource) throws HyracksDataException {
+ IIOManager ioManager = runtimeContextProvider.getIOManager();
+ FileReference file = ioManager.getFileRef(resource.getPath(), true);
+ LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(ioManager, file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields,
runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- new BaseOperationTracker(datasetID,
- runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
+ new BaseOperationTracker(datasetId(),
+ runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetId())),
runtimeContextProvider.getLSMIOScheduler(),
LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), -1, true);
return lsmBTree;
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadataFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadataFactory.java
new file mode 100644
index 0000000..61d19c3
--- /dev/null
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadataFactory.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.transaction.management.resource;
+
+import java.util.Map;
+
+import org.apache.asterix.common.transactions.Resource;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+public class ExternalBTreeLocalResourceMetadataFactory extends LSMBTreeLocalResourceMetadataFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public ExternalBTreeLocalResourceMetadataFactory(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+ int[] bloomFilterKeyFields, boolean isPrimary, int datasetID,
+ ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties) {
+ super(typeTraits, cmpFactories, bloomFilterKeyFields, isPrimary, datasetID, mergePolicyFactory,
+ mergePolicyProperties, null, null, null, null);
+ }
+
+ @Override
+ public Resource resource(int partition) {
+ return new ExternalBTreeLocalResourceMetadata(filterTypeTraits, filterCmpFactories, bloomFilterKeyFields,
+ isPrimary, datasetId, partition, mergePolicyFactory, mergePolicyProperties);
+ }
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
index 04006f1..3a486a5 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
@@ -18,24 +18,26 @@
*/
package org.apache.asterix.transaction.management.resource;
-import java.io.File;
import java.util.Map;
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.common.file.LocalResource;
/**
* The local resource for disk only lsm btree with buddy tree
*/
-public class ExternalBTreeWithBuddyLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
+public class ExternalBTreeWithBuddyLocalResourceMetadata extends Resource {
private static final long serialVersionUID = 1L;
@@ -45,10 +47,11 @@
private final Map<String, String> mergePolicyProperties;
private final int[] buddyBtreeFields;
- public ExternalBTreeWithBuddyLocalResourceMetadata(int datasetID, IBinaryComparatorFactory[] btreeCmpFactories,
+ public ExternalBTreeWithBuddyLocalResourceMetadata(int datasetID, int partition,
+ IBinaryComparatorFactory[] btreeCmpFactories,
ITypeTraits[] typeTraits, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, int[] buddyBtreeFields) {
- super(datasetID, null, null, null);
+ super(datasetID, partition, null, null, null);
this.btreeCmpFactories = btreeCmpFactories;
this.typeTraits = typeTraits;
this.mergePolicyFactory = mergePolicyFactory;
@@ -57,16 +60,17 @@
}
@Override
- public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) throws HyracksDataException {
- FileReference file = new FileReference(new File(filePath));
- return LSMBTreeUtils.createExternalBTreeWithBuddy(file, runtimeContextProvider.getBufferCache(),
+ public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider,
+ LocalResource resource) throws HyracksDataException {
+ IIOManager ioManager = runtimeContextProvider.getIOManager();
+ FileReference file = ioManager.getFileRef(resource.getPath(), true);
+ return LSMBTreeUtils.createExternalBTreeWithBuddy(ioManager, file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, btreeCmpFactories,
runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- new BaseOperationTracker(datasetID,
- runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
+ new BaseOperationTracker(datasetId(),
+ runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetId())),
runtimeContextProvider.getLSMIOScheduler(),
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), buddyBtreeFields, -1,
true);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadataFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadataFactory.java
new file mode 100644
index 0000000..836bcf4
--- /dev/null
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadataFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.transaction.management.resource;
+
+import java.util.Map;
+
+import org.apache.asterix.common.transactions.Resource;
+import org.apache.asterix.common.transactions.ResourceFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+public class ExternalBTreeWithBuddyLocalResourceMetadataFactory extends ResourceFactory {
+ private static final long serialVersionUID = 1L;
+ private final ITypeTraits[] typeTraits;
+ private final IBinaryComparatorFactory[] btreeCmpFactories;
+ private final ILSMMergePolicyFactory mergePolicyFactory;
+ private final Map<String, String> mergePolicyProperties;
+ private final int[] buddyBtreeFields;
+
+ public ExternalBTreeWithBuddyLocalResourceMetadataFactory(int datasetID,
+ IBinaryComparatorFactory[] btreeCmpFactories,
+ ITypeTraits[] typeTraits, ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, int[] buddyBtreeFields) {
+ super(datasetID, null, null, null);
+ this.btreeCmpFactories = btreeCmpFactories;
+ this.typeTraits = typeTraits;
+ this.mergePolicyFactory = mergePolicyFactory;
+ this.mergePolicyProperties = mergePolicyProperties;
+ this.buddyBtreeFields = buddyBtreeFields;
+ }
+
+ @Override
+ public Resource resource(int partition) {
+ return new ExternalBTreeWithBuddyLocalResourceMetadata(datasetId, partition, btreeCmpFactories, typeTraits,
+ mergePolicyFactory, mergePolicyProperties, buddyBtreeFields);
+ }
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
index 9ed15d2..7f6c4a3 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.transaction.management.resource;
-import java.io.File;
import java.util.Map;
import org.apache.asterix.common.context.BaseOperationTracker;
@@ -29,12 +28,14 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.apache.hyracks.storage.common.file.LocalResource;
/**
* The local resource class for disk only lsm r-tree
@@ -46,25 +47,26 @@
public ExternalRTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID,
- ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] btreeFields,
- boolean isPointMBR) {
+ int partition, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+ int[] btreeFields, boolean isPointMBR) {
super(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
- linearizeCmpFactory, datasetID, mergePolicyFactory, mergePolicyProperties, null, null, null,
+ linearizeCmpFactory, datasetID, partition, mergePolicyFactory, mergePolicyProperties, null, null, null,
btreeFields, null, isPointMBR);
}
@Override
- public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) throws HyracksDataException {
- FileReference file = new FileReference(new File(filePath));
+ public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider,
+ LocalResource resource) throws HyracksDataException {
+ IIOManager ioManager = runtimeContextProvider.getIOManager();
+ FileReference file = ioManager.getFileRef(resource.getPath(), true);
try {
- return LSMRTreeUtils.createExternalRTree(file, runtimeContextProvider.getBufferCache(),
+ return LSMRTreeUtils.createExternalRTree(ioManager, file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- new BaseOperationTracker(datasetID,
- runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
+ new BaseOperationTracker(datasetId(),
+ runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetId())),
runtimeContextProvider.getLSMIOScheduler(),
LSMRTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), linearizeCmpFactory,
btreeFields, -1, true, isPointMBR);
@@ -72,5 +74,4 @@
throw new HyracksDataException(e);
}
}
-
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadataFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadataFactory.java
new file mode 100644
index 0000000..076fef1
--- /dev/null
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadataFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.transaction.management.resource;
+
+import java.util.Map;
+
+import org.apache.asterix.common.transactions.Resource;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+
+public class ExternalRTreeLocalResourceMetadataFactory extends LSMRTreeLocalResourceMetadataFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public ExternalRTreeLocalResourceMetadataFactory(ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID,
+ ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+ int[] btreeFields, boolean isPointMBR) {
+ super(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ linearizeCmpFactory, datasetID, mergePolicyFactory, mergePolicyProperties, null, null, null,
+ btreeFields, null, isPointMBR);
+ }
+
+ @Override
+ public Resource resource(int partition) {
+ return new ExternalRTreeLocalResourceMetadata(filterTypeTraits, rtreeCmpFactories, btreeCmpFactories,
+ valueProviderFactories, rtreePolicyType, linearizeCmpFactory, datasetId, partition, mergePolicyFactory,
+ mergePolicyProperties, btreeFields, isPointMBR);
+ }
+
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
index b1db1f3..9c52241 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
@@ -18,22 +18,27 @@
*/
package org.apache.asterix.transaction.management.resource;
-import java.io.File;
+import java.util.List;
import java.util.Map;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.common.file.LocalResource;
-public class LSMBTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
+public class LSMBTreeLocalResourceMetadata extends Resource {
private static final long serialVersionUID = 1L;
@@ -46,10 +51,11 @@
protected final int[] btreeFields;
public LSMBTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields, boolean isPrimary, int datasetID, ILSMMergePolicyFactory mergePolicyFactory,
+ int[] bloomFilterKeyFields, boolean isPrimary, int datasetID, int partition,
+ ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields) {
- super(datasetID, filterTypeTraits, filterCmpFactories, filterFields);
+ super(datasetID, partition, filterTypeTraits, filterCmpFactories, filterFields);
this.typeTraits = typeTraits;
this.cmpFactories = cmpFactories;
this.bloomFilterKeyFields = bloomFilterKeyFields;
@@ -60,20 +66,38 @@
}
@Override
- public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) {
- FileReference file = new FileReference(new File(filePath));
+ public String toString() {
+ return new StringBuilder().append(" { \"").append(LSMBTreeLocalResourceMetadata.class.getName()).append(
+ "\" : {").append("\"datasetId\" : ").append(datasetId()).append(", \"partition\" : ").append(
+ partition()).append(" } ").append(" }").toString();
+ }
+
+ @Override
+ public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider,
+ LocalResource resource) throws HyracksDataException {
+ IIOManager ioManager = runtimeContextProvider.getIOManager();
+ FileReference file = ioManager.getFileRef(resource.getPath(), true);
+ List<IODeviceHandle> ioDevices = ioManager.getIODevices();
+ int ioDeviceNum = 0;
+ for (int i = 0; i < ioDevices.size(); i++) {
+ IODeviceHandle device = ioDevices.get(i);
+ if (device == file.getDeviceHandle()) {
+ ioDeviceNum = i;
+ break;
+ }
+ }
final IDatasetLifecycleManager datasetLifecycleManager = runtimeContextProvider.getDatasetLifecycleManager();
- LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(datasetLifecycleManager.getVirtualBufferCaches(datasetID,
+ LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(ioManager, datasetLifecycleManager.getVirtualBufferCaches(
+ datasetId(),
ioDeviceNum), file, runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(),
- typeTraits, cmpFactories, bloomFilterKeyFields, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ typeTraits, cmpFactories, bloomFilterKeyFields, runtimeContextProvider
+ .getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties, datasetLifecycleManager),
- isPrimary ? runtimeContextProvider.getLSMBTreeOperationTracker(datasetID)
- : new BaseOperationTracker(datasetID, datasetLifecycleManager.getDatasetInfo(datasetID)),
+ isPrimary ? runtimeContextProvider.getLSMBTreeOperationTracker(datasetId())
+ : new BaseOperationTracker(datasetId(), datasetLifecycleManager.getDatasetInfo(datasetId())),
runtimeContextProvider.getLSMIOScheduler(),
LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), isPrimary, filterTypeTraits,
filterCmpFactories, btreeFields, filterFields, true);
return lsmBTree;
}
-
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadataFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadataFactory.java
new file mode 100644
index 0000000..34d646d
--- /dev/null
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadataFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.transaction.management.resource;
+
+import java.util.Map;
+
+import org.apache.asterix.common.transactions.Resource;
+import org.apache.asterix.common.transactions.ResourceFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+public class LSMBTreeLocalResourceMetadataFactory extends ResourceFactory {
+
+ private static final long serialVersionUID = 1L;
+ protected final ITypeTraits[] typeTraits;
+ protected final IBinaryComparatorFactory[] cmpFactories;
+ protected final int[] bloomFilterKeyFields;
+ protected final boolean isPrimary;
+ protected final ILSMMergePolicyFactory mergePolicyFactory;
+ protected final Map<String, String> mergePolicyProperties;
+ protected final int[] btreeFields;
+
+ public LSMBTreeLocalResourceMetadataFactory(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+ int[] bloomFilterKeyFields, boolean isPrimary, int datasetID,
+ ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields) {
+ super(datasetID, filterTypeTraits, filterCmpFactories, filterFields);
+ this.typeTraits = typeTraits;
+ this.cmpFactories = cmpFactories;
+ this.bloomFilterKeyFields = bloomFilterKeyFields;
+ this.isPrimary = isPrimary;
+ this.mergePolicyFactory = mergePolicyFactory;
+ this.mergePolicyProperties = mergePolicyProperties;
+ this.btreeFields = btreeFields;
+ }
+
+ @Override
+ public Resource resource(int partition) {
+ return new LSMBTreeLocalResourceMetadata(typeTraits, cmpFactories, bloomFilterKeyFields, isPrimary, datasetId,
+ partition, mergePolicyFactory, mergePolicyProperties, typeTraits, filterCmpFactories, btreeFields,
+ filterFields);
+ }
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
index 87319b4..06bf298 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
@@ -24,17 +24,22 @@
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
+import org.apache.hyracks.storage.common.file.LocalResource;
-public class LSMInvertedIndexLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
+public class LSMInvertedIndexLocalResourceMetadata extends Resource {
private static final long serialVersionUID = 1L;
@@ -53,11 +58,11 @@
public LSMInvertedIndexLocalResourceMetadata(ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- boolean isPartitioned, int datasetID, ILSMMergePolicyFactory mergePolicyFactory,
+ boolean isPartitioned, int datasetID, int partition, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] invertedIndexFields, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
- super(datasetID, filterTypeTraits, filterCmpFactories, filterFields);
+ super(datasetID, partition, filterTypeTraits, filterCmpFactories, filterFields);
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
@@ -72,33 +77,44 @@
}
@Override
- public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) throws HyracksDataException {
+ public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider,
+ LocalResource resource) throws HyracksDataException {
+ IIOManager ioManager = runtimeContextProvider.getIOManager();
+ FileReference file = ioManager.getFileRef(resource.getPath(), true);
+ List<IODeviceHandle> ioDevices = ioManager.getIODevices();
+ int ioDeviceNum = 0;
+ for (int i = 0; i < ioDevices.size(); i++) {
+ IODeviceHandle device = ioDevices.get(i);
+ if (device == file.getDeviceHandle()) {
+ ioDeviceNum = i;
+ break;
+ }
+ }
List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager()
- .getVirtualBufferCaches(datasetID, ioDeviceNum);
+ .getVirtualBufferCaches(datasetId(), ioDeviceNum);
try {
if (isPartitioned) {
- return InvertedIndexUtils.createPartitionedLSMInvertedIndex(virtualBufferCaches,
+ return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches,
runtimeContextProvider.getFileMapManager(), invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, runtimeContextProvider.getBufferCache(),
- filePath, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ file.getAbsolutePath(), runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- new BaseOperationTracker(datasetID,
- runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
+ new BaseOperationTracker(datasetId(),
+ runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetId())),
runtimeContextProvider.getLSMIOScheduler(),
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(),
invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, true);
} else {
- return InvertedIndexUtils.createLSMInvertedIndex(virtualBufferCaches,
+ return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches,
runtimeContextProvider.getFileMapManager(), invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, runtimeContextProvider.getBufferCache(),
- filePath, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ file.getAbsolutePath(), runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- new BaseOperationTracker(datasetID,
- runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
+ new BaseOperationTracker(datasetId(),
+ runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetId())),
runtimeContextProvider.getLSMIOScheduler(),
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(),
invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadataFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadataFactory.java
new file mode 100644
index 0000000..a5a6c2a
--- /dev/null
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadataFactory.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.transaction.management.resource;
+
+import java.util.Map;
+
+import org.apache.asterix.common.transactions.Resource;
+import org.apache.asterix.common.transactions.ResourceFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+
+public class LSMInvertedIndexLocalResourceMetadataFactory extends ResourceFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final ITypeTraits[] invListTypeTraits;
+ private final IBinaryComparatorFactory[] invListCmpFactories;
+ private final ITypeTraits[] tokenTypeTraits;
+ private final IBinaryComparatorFactory[] tokenCmpFactories;
+ private final IBinaryTokenizerFactory tokenizerFactory;
+ private final boolean isPartitioned;
+ private final ILSMMergePolicyFactory mergePolicyFactory;
+ private final Map<String, String> mergePolicyProperties;
+ private final int[] invertedIndexFields;
+ private final int[] filterFieldsForNonBulkLoadOps;
+ private final int[] invertedIndexFieldsForNonBulkLoadOps;
+
+ public LSMInvertedIndexLocalResourceMetadataFactory(ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
+ boolean isPartitioned, int datasetID, ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] invertedIndexFields, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
+ super(datasetID, filterTypeTraits, filterCmpFactories, filterFields);
+ this.invListTypeTraits = invListTypeTraits;
+ this.invListCmpFactories = invListCmpFactories;
+ this.tokenTypeTraits = tokenTypeTraits;
+ this.tokenCmpFactories = tokenCmpFactories;
+ this.tokenizerFactory = tokenizerFactory;
+ this.isPartitioned = isPartitioned;
+ this.mergePolicyFactory = mergePolicyFactory;
+ this.mergePolicyProperties = mergePolicyProperties;
+ this.invertedIndexFields = invertedIndexFields;
+ this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
+ this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
+ }
+
+ @Override
+ public Resource resource(int partition) {
+ return new LSMInvertedIndexLocalResourceMetadata(invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, tokenizerFactory, isPartitioned, datasetId, partition, mergePolicyFactory,
+ mergePolicyProperties, filterTypeTraits, filterCmpFactories, invertedIndexFields,
+ filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
+ }
+
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index 535448c..73c2dd9 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -18,18 +18,20 @@
*/
package org.apache.asterix.transaction.management.resource;
-import java.io.File;
import java.util.List;
import java.util.Map;
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
@@ -37,8 +39,9 @@
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.apache.hyracks.storage.common.file.LocalResource;
-public class LSMRTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
+public class LSMRTreeLocalResourceMetadata extends Resource {
private static final long serialVersionUID = 1L;
@@ -57,10 +60,10 @@
public LSMRTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID,
- ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+ int partition, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] rtreeFields,
int[] btreeFields, int[] filterFields, boolean isPointMBR) {
- super(datasetID, filterTypeTraits, filterCmpFactories, filterFields);
+ super(datasetID, partition, filterTypeTraits, filterCmpFactories, filterFields);
this.typeTraits = typeTraits;
this.rtreeCmpFactories = rtreeCmpFactories;
this.btreeCmpFactories = btreeCmpFactories;
@@ -75,19 +78,29 @@
}
@Override
- public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) throws HyracksDataException {
- FileReference file = new FileReference(new File(filePath));
+ public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider,
+ LocalResource resource) throws HyracksDataException {
+ IIOManager ioManager = runtimeContextProvider.getIOManager();
+ FileReference file = ioManager.getFileRef(resource.getPath(), true);
+ List<IODeviceHandle> ioDevices = ioManager.getIODevices();
+ int ioDeviceNum = 0;
+ for (int i = 0; i < ioDevices.size(); i++) {
+ IODeviceHandle device = ioDevices.get(i);
+ if (device == file.getDeviceHandle()) {
+ ioDeviceNum = i;
+ break;
+ }
+ }
List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager()
- .getVirtualBufferCaches(datasetID, ioDeviceNum);
+ .getVirtualBufferCaches(datasetId(), ioDeviceNum);
try {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches, file,
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, file,
runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- new BaseOperationTracker(datasetID,
- runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
+ new BaseOperationTracker(datasetId(),
+ runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetId())),
runtimeContextProvider.getLSMIOScheduler(),
LSMRTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), linearizeCmpFactory,
rtreeFields, filterTypeTraits, filterCmpFactories, filterFields, true, isPointMBR);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadataFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadataFactory.java
new file mode 100644
index 0000000..0ecf0aa
--- /dev/null
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadataFactory.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.transaction.management.resource;
+
+import java.util.Map;
+
+import org.apache.asterix.common.transactions.Resource;
+import org.apache.asterix.common.transactions.ResourceFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+
+public class LSMRTreeLocalResourceMetadataFactory extends ResourceFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ protected final ITypeTraits[] typeTraits;
+ protected final IBinaryComparatorFactory[] rtreeCmpFactories;
+ protected final IBinaryComparatorFactory[] btreeCmpFactories;
+ protected final IPrimitiveValueProviderFactory[] valueProviderFactories;
+ protected final RTreePolicyType rtreePolicyType;
+ protected final ILinearizeComparatorFactory linearizeCmpFactory;
+ protected final ILSMMergePolicyFactory mergePolicyFactory;
+ protected final Map<String, String> mergePolicyProperties;
+ protected final int[] rtreeFields;
+ protected final int[] btreeFields;
+ protected final boolean isPointMBR;
+
+ public LSMRTreeLocalResourceMetadataFactory(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID,
+ ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] rtreeFields,
+ int[] btreeFields, int[] filterFields, boolean isPointMBR) {
+ super(datasetID, filterTypeTraits, filterCmpFactories, filterFields);
+ this.typeTraits = typeTraits;
+ this.rtreeCmpFactories = rtreeCmpFactories;
+ this.btreeCmpFactories = btreeCmpFactories;
+ this.valueProviderFactories = valueProviderFactories;
+ this.rtreePolicyType = rtreePolicyType;
+ this.linearizeCmpFactory = linearizeCmpFactory;
+ this.mergePolicyFactory = mergePolicyFactory;
+ this.mergePolicyProperties = mergePolicyProperties;
+ this.rtreeFields = rtreeFields;
+ this.btreeFields = btreeFields;
+ this.isPointMBR = isPointMBR;
+ }
+
+ @Override
+ public Resource resource(int partition) {
+ return new LSMRTreeLocalResourceMetadata(typeTraits, rtreeCmpFactories, btreeCmpFactories,
+ valueProviderFactories, rtreePolicyType, linearizeCmpFactory, datasetId, partition, mergePolicyFactory,
+ mergePolicyProperties, filterTypeTraits, filterCmpFactories, rtreeFields, btreeFields, filterFields,
+ isPointMBR);
+ }
+}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactory.java
index 33e5bf2..8ab99fb 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactory.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactory.java
@@ -18,24 +18,24 @@
*/
package org.apache.asterix.transaction.management.resource;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
+import org.apache.asterix.common.transactions.IResourceFactory;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.hyracks.storage.common.file.ILocalResourceFactory;
import org.apache.hyracks.storage.common.file.LocalResource;
public class PersistentLocalResourceFactory implements ILocalResourceFactory {
- private final ILocalResourceMetadata localResourceMetadata;
+ private final IResourceFactory applicationResourceFactory;
private final int resourceType;
- public PersistentLocalResourceFactory(ILocalResourceMetadata localResourceMetadata, int resourceType) {
- this.localResourceMetadata = localResourceMetadata;
+ public PersistentLocalResourceFactory(IResourceFactory applicationResourceFactory, int resourceType) {
+ this.applicationResourceFactory = applicationResourceFactory;
this.resourceType = resourceType;
}
@Override
- public LocalResource createLocalResource(long resourceId, String resourceName, int partition, int storageVersion,
- String resourcePath) {
- return new LocalResource(resourceId, resourceName, partition, resourcePath, resourceType, storageVersion,
- localResourceMetadata);
+ public LocalResource createLocalResource(long id, String name, int version, int partition) {
+ Resource resource = applicationResourceFactory.resource(partition);
+ return new LocalResource(id, name, resourceType, version, resource);
}
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java
index 75d0769..c306699 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java
@@ -18,23 +18,23 @@
*/
package org.apache.asterix.transaction.management.resource;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
+import org.apache.asterix.common.transactions.IResourceFactory;
import org.apache.hyracks.storage.common.file.ILocalResourceFactory;
import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider;
public class PersistentLocalResourceFactoryProvider implements ILocalResourceFactoryProvider {
private static final long serialVersionUID = 1L;
- private final ILocalResourceMetadata localResourceMetadata;
- private final int resourceType;
+ private final IResourceFactory factory;
+ private final int type;
- public PersistentLocalResourceFactoryProvider(ILocalResourceMetadata localResourceMetadata, int resourceType) {
- this.localResourceMetadata = localResourceMetadata;
- this.resourceType = resourceType;
+ public PersistentLocalResourceFactoryProvider(IResourceFactory factory, int type) {
+ this.factory = factory;
+ this.type = type;
}
@Override
public ILocalResourceFactory getLocalResourceFactory() {
- return new PersistentLocalResourceFactory(localResourceMetadata, resourceType);
+ return new PersistentLocalResourceFactory(factory, type);
}
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 006eac7..9ebb8e7 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -33,6 +33,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.ConcurrentHashMap;
@@ -47,10 +48,12 @@
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationJobType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrame;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
import org.apache.hyracks.storage.common.file.LocalResource;
@@ -64,13 +67,14 @@
public static final String METADATA_FILE_NAME = ".metadata";
// Private constants
private static final Logger LOGGER = Logger.getLogger(PersistentLocalResourceRepository.class.getName());
- private static final String STORAGE_METADATA_DIRECTORY = "asterix_root_metadata";
- private static final String STORAGE_METADATA_FILE_NAME_PREFIX = ".asterix_root_metadata";
+ private static final String STORAGE_METADATA_DIRECTORY = "root_metadata";
+ private static final String STORAGE_METADATA_FILE_NAME_PREFIX = ".root_metadata";
private static final long STORAGE_LOCAL_RESOURCE_ID = -4321;
private static final int MAX_CACHED_RESOURCES = 1000;
- private static final FilenameFilter METADATA_FILES_FILTER =
- (File dir, String name) -> name.equalsIgnoreCase(METADATA_FILE_NAME);
+ private static final FilenameFilter METADATA_FILES_FILTER = (File dir, String name) -> name.equalsIgnoreCase(
+ METADATA_FILE_NAME);
// Finals
+ private final IOManager ioManager;
private final String[] mountPoints;
private final String nodeId;
private final Cache<String, LocalResource> resourceCache;
@@ -83,13 +87,14 @@
private IReplicationManager replicationManager;
private Set<Integer> nodeInactivePartitions;
- public PersistentLocalResourceRepository(List<IODeviceHandle> devices, String nodeId,
+ public PersistentLocalResourceRepository(IOManager ioManager, List<IODeviceHandle> devices, String nodeId,
AsterixMetadataProperties metadataProperties) throws HyracksDataException {
+ this.ioManager = ioManager;
mountPoints = new String[devices.size()];
this.nodeId = nodeId;
this.clusterPartitions = metadataProperties.getClusterPartitions();
for (int i = 0; i < mountPoints.length; i++) {
- String mountPoint = devices.get(i).getPath().getPath();
+ String mountPoint = devices.get(i).getMount().getPath();
File mountPointDir = new File(mountPoint);
if (!mountPointDir.exists()) {
throw new HyracksDataException(mountPointDir.getAbsolutePath() + " doesn't exist.");
@@ -112,13 +117,16 @@
}
}
- private static String getStorageMetadataDirPath(String mountPoint, String nodeId, int ioDeviceId) {
- return mountPoint + STORAGE_METADATA_DIRECTORY + File.separator + nodeId + "_" + "iodevice" + ioDeviceId;
- }
-
- private static File getStorageMetadataBaseDir(File storageMetadataFile) {
- //STORAGE_METADATA_DIRECTORY / Node Id / STORAGE_METADATA_FILE_NAME_PREFIX
- return storageMetadataFile.getParentFile().getParentFile();
+ @Override
+ public String toString() {
+ StringBuilder aString = new StringBuilder().append(PersistentLocalResourceRepository.class.getSimpleName())
+ .append(Character.LINE_SEPARATOR).append(ioManager.getClass().getSimpleName()).append(':').append(
+ Character.LINE_SEPARATOR).append(ioManager.toString()).append(Character.LINE_SEPARATOR).append(
+ "Cached Resources:").append(Character.LINE_SEPARATOR);
+ for (Entry<String, LocalResource> pair : resourceCache.asMap().entrySet()) {
+ aString.append(pair.getKey()).append("->").append(pair.getValue()).append(Character.LINE_SEPARATOR);
+ }
+ return aString.toString();
}
public void initializeNewUniverse(String storageRootDirName) throws HyracksDataException {
@@ -134,8 +142,8 @@
* This file could potentially store more information, otherwise no need to keep it.
*/
for (int i = 0; i < mountPoints.length; i++) {
- File storageMetadataFile = getStorageMetadataFile(mountPoints[i], nodeId, i);
- File storageMetadataDir = storageMetadataFile.getParentFile();
+ FileReference storageMetadataFile = getStorageMetadataFile(ioManager, nodeId, i);
+ File storageMetadataDir = storageMetadataFile.getFile().getParentFile();
//make dirs for the storage metadata file
boolean success = storageMetadataDir.mkdirs();
if (!success) {
@@ -156,8 +164,7 @@
}
LocalResource rootLocalResource = new LocalResource(STORAGE_LOCAL_RESOURCE_ID,
- storageMetadataFile.getAbsolutePath(), 0, storageMetadataFile.getAbsolutePath(), 0,
- LIFOMetaDataFrame.VERSION, storageRootDirPath);
+ storageMetadataFile.getRelativePath(), 0, LIFOMetaDataFrame.VERSION, storageRootDirPath);
insert(rootLocalResource);
LOGGER.log(Level.INFO, "created the root-metadata-file: " + storageMetadataFile.getAbsolutePath());
}
@@ -165,13 +172,13 @@
}
@Override
- public LocalResource getResourceByPath(String path) throws HyracksDataException {
- LocalResource resource = resourceCache.getIfPresent(path);
+ public LocalResource get(String relativePath) throws HyracksDataException {
+ LocalResource resource = resourceCache.getIfPresent(relativePath);
if (resource == null) {
- File resourceFile = getLocalResourceFileByName(path);
- if (resourceFile.exists()) {
- resource = readLocalResource(resourceFile);
- resourceCache.put(path, resource);
+ FileReference resourceFile = getLocalResourceFileByName(ioManager, relativePath);
+ if (resourceFile.getFile().exists()) {
+ resource = readLocalResource(resourceFile.getFile());
+ resourceCache.put(relativePath, resource);
}
}
return resource;
@@ -179,18 +186,19 @@
@Override
public synchronized void insert(LocalResource resource) throws HyracksDataException {
- File resourceFile = new File(getFileName(resource.getResourcePath(), resource.getResourceId()));
- if (resourceFile.exists()) {
+ String relativePath = getFileName(resource.getPath(), resource.getId());
+ FileReference resourceFile = ioManager.getFileRef(relativePath, true);
+ if (resourceFile.getFile().exists()) {
throw new HyracksDataException("Duplicate resource: " + resourceFile.getAbsolutePath());
} else {
- resourceFile.getParentFile().mkdirs();
+ resourceFile.getFile().getParentFile().mkdirs();
}
- if (resource.getResourceId() != STORAGE_LOCAL_RESOURCE_ID) {
- resourceCache.put(resource.getResourcePath(), resource);
+ if (resource.getId() != STORAGE_LOCAL_RESOURCE_ID) {
+ resourceCache.put(resource.getPath(), resource);
}
- try (FileOutputStream fos = new FileOutputStream(resourceFile);
+ try (FileOutputStream fos = new FileOutputStream(resourceFile.getFile());
ObjectOutputStream oosToFos = new ObjectOutputStream(fos)) {
oosToFos.writeObject(resource);
oosToFos.flush();
@@ -199,21 +207,22 @@
}
//if replication enabled, send resource metadata info to remote nodes
- if (isReplicationEnabled && resource.getResourceId() != STORAGE_LOCAL_RESOURCE_ID) {
- String filePath = getFileName(resource.getResourcePath(), resource.getResourceId());
+ if (isReplicationEnabled && resource.getId() != STORAGE_LOCAL_RESOURCE_ID) {
+ String filePath = getFileName(resource.getPath(), resource.getId());
createReplicationJob(ReplicationOperation.REPLICATE, filePath);
}
}
@Override
- public synchronized void deleteResourceByPath(String resourcePath) throws HyracksDataException {
- File resourceFile = getLocalResourceFileByName(resourcePath);
- if (resourceFile.exists()) {
+ public synchronized void delete(String relativePath) throws HyracksDataException {
+ FileReference resourceFile = getLocalResourceFileByName(ioManager, relativePath);
+ if (resourceFile.getFile().exists()) {
resourceFile.delete();
- resourceCache.invalidate(resourcePath);
+ resourceCache.invalidate(relativePath);
//if replication enabled, delete resource from remote replicas
- if (isReplicationEnabled && !resourceFile.getName().startsWith(STORAGE_METADATA_FILE_NAME_PREFIX)) {
+ if (isReplicationEnabled && !resourceFile.getFile().getName().startsWith(
+ STORAGE_METADATA_FILE_NAME_PREFIX)) {
createReplicationJob(ReplicationOperation.DELETE, resourceFile.getAbsolutePath());
}
} else {
@@ -221,8 +230,10 @@
}
}
- private static File getLocalResourceFileByName(String resourcePath) {
- return new File(resourcePath + File.separator + METADATA_FILE_NAME);
+ private static FileReference getLocalResourceFileByName(IOManager ioManager, String resourcePath)
+ throws HyracksDataException {
+ String fileName = resourcePath + File.separator + METADATA_FILE_NAME;
+ return ioManager.getFileRef(fileName, true);
}
public Map<Long, LocalResource> loadAndGetAllResources() throws HyracksDataException {
@@ -230,7 +241,7 @@
//This could be fixed by traversing all resources on disk until the required resource is found.
Map<Long, LocalResource> resourcesMap = new HashMap<>();
for (int i = 0; i < mountPoints.length; i++) {
- File storageRootDir = getStorageRootDirectoryIfExists(mountPoints[i], nodeId, i);
+ File storageRootDir = getStorageRootDirectoryIfExists(ioManager, nodeId, i);
if (storageRootDir == null) {
continue;
}
@@ -265,18 +276,18 @@
if (metadataFiles != null) {
for (File metadataFile : metadataFiles) {
LocalResource localResource = readLocalResource(metadataFile);
- resourcesMap.put(localResource.getResourceId(), localResource);
+ resourcesMap.put(localResource.getId(), localResource);
}
}
}
}
@Override
- public long getMaxResourceID() throws HyracksDataException {
+ public long maxId() throws HyracksDataException {
long maxResourceId = 0;
for (int i = 0; i < mountPoints.length; i++) {
- File storageRootDir = getStorageRootDirectoryIfExists(mountPoints[i], nodeId, i);
+ File storageRootDir = getStorageRootDirectoryIfExists(ioManager, nodeId, i);
if (storageRootDir == null) {
continue;
}
@@ -316,7 +327,7 @@
if (metadataFiles != null) {
for (File metadataFile : metadataFiles) {
LocalResource localResource = readLocalResource(metadataFile);
- maxResourceId = Math.max(maxResourceId, localResource.getResourceId());
+ maxResourceId = Math.max(maxResourceId, localResource.getId());
}
}
}
@@ -388,14 +399,14 @@
*/
public void deleteStorageData(boolean deleteStorageMetadata) throws IOException {
for (int i = 0; i < mountPoints.length; i++) {
- File storageDir = getStorageRootDirectoryIfExists(mountPoints[i], nodeId, i);
+ File storageDir = getStorageRootDirectoryIfExists(ioManager, nodeId, i);
if (storageDir != null && storageDir.isDirectory()) {
FileUtils.deleteDirectory(storageDir);
}
if (deleteStorageMetadata) {
//delete the metadata root directory
- File storageMetadataFile = getStorageMetadataFile(mountPoints[i], nodeId, i);
- File storageMetadataDir = getStorageMetadataBaseDir(storageMetadataFile);
+ FileReference storageMetadataFile = getStorageMetadataFile(ioManager, nodeId, i);
+ File storageMetadataDir = storageMetadataFile.getFile().getParentFile().getParentFile();
if (storageMetadataDir.exists() && storageMetadataDir.isDirectory()) {
FileUtils.deleteDirectory(storageMetadataDir);
}
@@ -409,10 +420,12 @@
* @param ioDeviceId
* @return A file reference to the storage metadata file.
*/
- private static File getStorageMetadataFile(String mountPoint, String nodeId, int ioDeviceId) {
- String storageMetadataFileName = getStorageMetadataDirPath(mountPoint, nodeId, ioDeviceId) + File.separator
+ private static FileReference getStorageMetadataFile(IOManager ioManager, String nodeId,
+ int ioDeviceId) {
+ String storageMetadataFileName = STORAGE_METADATA_DIRECTORY + File.separator + nodeId + "_" + "iodevice"
+ + ioDeviceId + File.separator
+ STORAGE_METADATA_FILE_NAME_PREFIX;
- return new File(storageMetadataFileName);
+ return new FileReference(ioManager.getIODevices().get(ioDeviceId), storageMetadataFileName);
}
/**
@@ -422,13 +435,14 @@
* @return A file reference to the storage root directory if exists, otherwise null.
* @throws HyracksDataException
*/
- public static File getStorageRootDirectoryIfExists(String mountPoint, String nodeId, int ioDeviceId)
+ public static File getStorageRootDirectoryIfExists(IOManager ioManager, String nodeId,
+ int ioDeviceId)
throws HyracksDataException {
File storageRootDir = null;
- File storageMetadataFile = getStorageMetadataFile(mountPoint, nodeId, ioDeviceId);
- if (storageMetadataFile.exists()) {
- LocalResource rootLocalResource = readLocalResource(storageMetadataFile);
- String storageRootDirPath = (String) rootLocalResource.getResourceObject();
+ FileReference storageMetadataFile = getStorageMetadataFile(ioManager, nodeId, ioDeviceId);
+ if (storageMetadataFile.getFile().exists()) {
+ LocalResource rootLocalResource = readLocalResource(storageMetadataFile.getFile());
+ String storageRootDirPath = (String) rootLocalResource.getResource();
Path path = Paths.get(storageRootDirPath);
if (Files.exists(path)) {
storageRootDir = new File(storageRootDirPath);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
index d62abc5..1c28e17 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
@@ -20,16 +20,16 @@
import org.apache.asterix.common.config.AsterixMetadataProperties;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
public class PersistentLocalResourceRepositoryFactory implements ILocalResourceRepositoryFactory {
- private final IIOManager ioManager;
+ private final IOManager ioManager;
private final String nodeId;
private final AsterixMetadataProperties metadataProperties;
- public PersistentLocalResourceRepositoryFactory(IIOManager ioManager, String nodeId,
+ public PersistentLocalResourceRepositoryFactory(IOManager ioManager, String nodeId,
AsterixMetadataProperties metadataProperties) {
this.ioManager = ioManager;
this.nodeId = nodeId;
@@ -38,6 +38,6 @@
@Override
public ILocalResourceRepository createRepository() throws HyracksDataException {
- return new PersistentLocalResourceRepository(ioManager.getIODevices(), nodeId, metadataProperties);
+ return new PersistentLocalResourceRepository(ioManager, ioManager.getIODevices(), nodeId, metadataProperties);
}
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
index 286fc16..1f8d698 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
@@ -48,7 +48,6 @@
import java.util.logging.Logger;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.config.AsterixMetadataProperties;
import org.apache.asterix.common.config.ClusterProperties;
@@ -63,6 +62,7 @@
import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.common.transactions.ITransactionContext;
import org.apache.asterix.common.transactions.LogType;
+import org.apache.asterix.common.transactions.Resource;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.asterix.transaction.management.service.logging.LogManager;
import org.apache.asterix.transaction.management.service.transaction.TransactionManagementConstants;
@@ -214,7 +214,7 @@
int abortLogCount = 0;
int jobId = -1;
- Set<Integer> winnerJobSet = new HashSet<Integer>();
+ Set<Integer> winnerJobSet = new HashSet<>();
jobId2WinnerEntitiesMap = new HashMap<>();
//set log reader to the lowWaterMarkLsn
@@ -300,7 +300,7 @@
long LSN = -1;
ILSMIndex index = null;
LocalResource localResource = null;
- ILocalResourceMetadata localResourceMetadata = null;
+ Resource localResourceMetadata = null;
boolean foundWinner = false;
JobEntityCommits jobEntityWinners = null;
@@ -308,7 +308,7 @@
IDatasetLifecycleManager datasetLifecycleManager = appRuntimeContext.getDatasetLifecycleManager();
Map<Long, LocalResource> resourcesMap = localResourceRepository.loadAndGetAllResources();
- Map<Long, Long> resourceId2MaxLSNMap = new HashMap<Long, Long>();
+ Map<Long, Long> resourceId2MaxLSNMap = new HashMap<>();
TxnId tempKeyTxnId = new TxnId(-1, -1, -1, null, -1, false);
ILogRecord logRecord = null;
@@ -361,20 +361,13 @@
//if index is not registered into IndexLifeCycleManager,
//create the index using LocalMetadata stored in LocalResourceRepository
//get partition path in this node
- String partitionIODevicePath =
- localResourceRepository.getPartitionPath(localResource.getPartition());
- String resourceAbsolutePath =
- partitionIODevicePath + File.separator + localResource.getResourceName();
- localResource.setResourcePath(resourceAbsolutePath);
- index = (ILSMIndex) datasetLifecycleManager.get(resourceAbsolutePath);
+ localResourceMetadata = (Resource) localResource.getResource();
+ index = (ILSMIndex) datasetLifecycleManager.get(localResource.getPath());
if (index == null) {
//#. create index instance and register to indexLifeCycleManager
- localResourceMetadata = (ILocalResourceMetadata) localResource.getResourceObject();
- index = localResourceMetadata.createIndexInstance(appRuntimeContext,
- resourceAbsolutePath, localResource.getPartition(),
- localResourceRepository.getIODeviceNum(localResource.getPartition()));
- datasetLifecycleManager.register(resourceAbsolutePath, index);
- datasetLifecycleManager.open(resourceAbsolutePath);
+ index = localResourceMetadata.createIndexInstance(appRuntimeContext, localResource);
+ datasetLifecycleManager.register(localResource.getPath(), index);
+ datasetLifecycleManager.open(localResource.getPath());
//#. get maxDiskLastLSN
ILSMIndex lsmIndex = index;
@@ -383,7 +376,7 @@
((AbstractLSMIOOperationCallback) lsmIndex.getIOOperationCallback())
.getComponentLSN(lsmIndex.getImmutableComponents());
} catch (HyracksDataException e) {
- datasetLifecycleManager.close(resourceAbsolutePath);
+ datasetLifecycleManager.close(localResource.getPath());
throw e;
}
@@ -419,7 +412,7 @@
//close all indexes
Set<Long> resourceIdList = resourceId2MaxLSNMap.keySet();
for (long r : resourceIdList) {
- datasetLifecycleManager.close(resourcesMap.get(r).getResourcePath());
+ datasetLifecycleManager.close(resourcesMap.get(r).getPath());
}
}
}
@@ -602,7 +595,7 @@
throw new FileNotFoundException("Checkpoint file is not found");
}
- List<CheckpointObject> checkpointObjectList = new ArrayList<CheckpointObject>();
+ List<CheckpointObject> checkpointObjectList = new ArrayList<>();
for (File file : prevCheckpointFiles) {
try (FileInputStream fis = new FileInputStream(file);
ObjectInputStream oisFromFis = new ObjectInputStream(fis)) {
@@ -720,7 +713,7 @@
LOGGER.info("collecting loser transaction's LSNs from " + firstLSN + " to " + lastLSN);
}
- Map<TxnId, List<Long>> jobLoserEntity2LSNsMap = new HashMap<TxnId, List<Long>>();
+ Map<TxnId, List<Long>> jobLoserEntity2LSNsMap = new HashMap<>();
TxnId tempKeyTxnId = new TxnId(-1, -1, -1, null, -1, false);
int updateLogCount = 0;
int entityCommitLogCount = 0;
@@ -758,7 +751,7 @@
if (undoLSNSet == null) {
loserEntity = new TxnId(logJobId, logRecord.getDatasetId(), logRecord.getPKHashValue(),
logRecord.getPKValue(), logRecord.getPKValueSize(), true);
- undoLSNSet = new LinkedList<Long>();
+ undoLSNSet = new LinkedList<>();
jobLoserEntity2LSNsMap.put(loserEntity, undoLSNSet);
}
undoLSNSet.add(currentLSN);
@@ -876,8 +869,10 @@
private static void redo(ILogRecord logRecord, IDatasetLifecycleManager datasetLifecycleManager) {
try {
+ int datasetId = logRecord.getDatasetId();
+ long resourceId = logRecord.getResourceId();
ILSMIndex index =
- (ILSMIndex) datasetLifecycleManager.getIndex(logRecord.getDatasetId(), logRecord.getResourceId());
+ (ILSMIndex) datasetLifecycleManager.getIndex(datasetId, resourceId);
ILSMIndexAccessor indexAccessor =
index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
if (logRecord.getNewOp() == IndexOperation.INSERT.ordinal()) {
@@ -888,7 +883,6 @@
throw new IllegalStateException("Unsupported OperationType: " + logRecord.getNewOp());
}
} catch (Exception e) {
- e.printStackTrace();
throw new IllegalStateException("Failed to redo", e);
}
}
@@ -896,8 +890,8 @@
private class JobEntityCommits {
private static final String PARTITION_FILE_NAME_SEPARATOR = "_";
private final int jobId;
- private final Set<TxnId> cachedEntityCommitTxns = new HashSet<TxnId>();
- private final List<File> jobEntitCommitOnDiskPartitionsFiles = new ArrayList<File>();
+ private final Set<TxnId> cachedEntityCommitTxns = new HashSet<>();
+ private final List<File> jobEntitCommitOnDiskPartitionsFiles = new ArrayList<>();
//a flag indicating whether all the the commits for this jobs have been added.
private boolean preparedForSearch = false;
private TxnId winnerEntity = null;
@@ -975,7 +969,7 @@
* @return partitions that have a max LSN > logLSN
*/
public ArrayList<File> getCandidiatePartitions(long logLSN) {
- ArrayList<File> candidiatePartitions = new ArrayList<File>();
+ ArrayList<File> candidiatePartitions = new ArrayList<>();
for (File partition : jobEntitCommitOnDiskPartitionsFiles) {
String partitionName = partition.getName();
//entity commit log must come after the update log, therefore, consider only partitions with max LSN > logLSN
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
index 4afcda8..9347947 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
@@ -18,7 +18,7 @@
*/
package org.apache.hyracks.algebricks.core.algebra.properties;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class FileSplitDomain implements INodeDomain {
@@ -44,7 +44,7 @@
} else {
sb.append(", ");
}
- sb.append(fs.getNodeName() + ":" + fs.getLocalFile());
+ sb.append(fs.getNodeName() + ":" + fs.getPath());
}
sb.append(']');
return sb.toString();
diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
index fe3e1d5..d01bae5 100644
--- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
+++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
@@ -20,8 +20,7 @@
import java.io.File;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class FileSplitUtils {
public static FileSplit[] parseFileSplits(String fileSplits) {
@@ -33,7 +32,7 @@
if (idx < 0) {
throw new IllegalArgumentException("File split " + s + " not well formed");
}
- fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(new File(s.substring(idx + 1))));
+ fSplits[i] = new FileSplit(s.substring(0, idx), new File(s.substring(idx + 1)).getAbsolutePath(), false);
}
return fSplits;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
index d159f60..d0d94ac 100644
--- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
+++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
@@ -22,7 +22,7 @@
import org.apache.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
import org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class PigletFileDataSink implements IDataSink {
private String file;
diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
index cb62a34..6a675fc 100644
--- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
+++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
@@ -31,7 +31,7 @@
import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
public class PigletFileDataSource implements IDataSource<String> {
private final String file;
diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
index 7bcb1d6..37906c1 100644
--- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
@@ -47,6 +47,8 @@
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
@@ -58,7 +60,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
@@ -66,7 +67,7 @@
private static final Map<FunctionIdentifier, PigletFunction> FN_MAP;
static {
- Map<FunctionIdentifier, PigletFunction> map = new HashMap<FunctionIdentifier, PigletFunction>();
+ Map<FunctionIdentifier, PigletFunction> map = new HashMap<>();
map.put(AlgebricksBuiltinFunctions.EQ, new PigletFunction(AlgebricksBuiltinFunctions.EQ));
@@ -129,7 +130,7 @@
IOperatorDescriptor scanner = new FileScanOperatorDescriptor(jobSpec, fsp, tpf, rDesc);
AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
- return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
+ return new Pair<>(scanner, constraint);
}
@Override
@@ -142,10 +143,15 @@
for (int i = 0; i < fileSplits.length; ++i) {
locations[i] = fileSplits[i].getNodeName();
}
- IPushRuntimeFactory prf = new SinkWriterRuntimeFactory(printColumns, printerFactories,
- fileSplits[0].getLocalFile().getFile(), PrinterBasedWriterFactory.INSTANCE, inputDesc);
+ IPushRuntimeFactory prf;
+ try {
+ prf = new SinkWriterRuntimeFactory(printColumns, printerFactories,
+ fileSplits[0].getFile(null), PrinterBasedWriterFactory.INSTANCE, inputDesc);
AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
- return new Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint>(prf, constraint);
+ return new Pair<>(prf, constraint);
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
}
@Override
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
index 1276518..af6ddca 100644
--- a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
+++ b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
@@ -64,7 +64,7 @@
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
@@ -80,7 +80,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.LineFileWriteOperatorDescriptor;
import org.apache.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
@@ -181,8 +180,8 @@
// the scanner
FileSplit[] intFileSplits = new FileSplit[1];
- intFileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
- new FileReference(new File("data/simple/int-part1.tbl")));
+ intFileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/simple/int-part1.tbl")
+ .getAbsolutePath(), false);
IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(intFileSplits);
RecordDescriptor intScannerDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
@@ -264,8 +263,8 @@
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
- fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
- new FileReference(new File("data/tpch0.001/customer.tbl")));
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/tpch0.001/customer.tbl")
+ .getAbsolutePath(), false);
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -347,7 +346,7 @@
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
- new FileReference(new File("data/tpch0.001/customer-part1.tbl")));
+ new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false);
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),
@@ -399,7 +398,7 @@
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
- new FileReference(new File("data/tpch0.001/customer.tbl")));
+ new File("data/tpch0.001/customer.tbl").getAbsolutePath(), false);
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),
@@ -584,7 +583,7 @@
}
FileSplit[] inputSplits = new FileSplit[] {
- new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(inputFile)) };
+ new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFile.getAbsolutePath(), false) };
DelimitedDataTupleParserFactory stringParser = new DelimitedDataTupleParserFactory(
new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE }, '\u0000');
@@ -605,7 +604,7 @@
IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length];
for (int i = 0; i < outputArity; i++) {
outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] {
- new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(outputFile[i])) });
+ new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, outputFile[i].getAbsolutePath(), false) });
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i],
new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
}
@@ -643,7 +642,7 @@
}
FileSplit[] inputSplits = new FileSplit[] {
- new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(inputFiles[0])) };
+ new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFiles[0].getAbsolutePath(), false) };
IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(inputSplits);
RecordDescriptor scannerDesc = new RecordDescriptor(
@@ -667,7 +666,7 @@
IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length];
for (int i = 0; i < outputArity; i++) {
outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] {
- new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(outputFile[i])) });
+ new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, outputFile[i].getAbsolutePath(), false) });
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i],
new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
}
@@ -693,8 +692,8 @@
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
- fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
- new FileReference(new File("data/tpch0.001/nation.tbl")));
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/tpch0.001/nation.tbl")
+ .getAbsolutePath(), false);
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),
@@ -803,7 +802,7 @@
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
- new FileReference(new File("data/tpch0.001/customer.tbl")));
+ new File("data/tpch0.001/customer.tbl").getAbsolutePath(), false);
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index 8893adb..12a138a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -21,20 +21,21 @@
import java.io.File;
import java.io.Serializable;
+/**
+ * A device handle and a relative path.
+ * Used to identify a file in the local Node Controller.
+ * Only used for files which are stored inside an IO device.
+ */
public final class FileReference implements Serializable {
private static final long serialVersionUID = 1L;
-
private final File file;
private final IODeviceHandle dev;
+ private final String path;
- public FileReference(IODeviceHandle dev, String devRelPath) {
- file = new File(dev.getPath(), devRelPath);
+ public FileReference(IODeviceHandle dev, String path) {
+ file = new File(dev.getMount(), path);
this.dev = dev;
- }
-
- public FileReference(File file) {
- this.file = file;
- this.dev = null;
+ this.path = path;
}
public File getFile() {
@@ -63,7 +64,26 @@
return file.hashCode();
}
- public void delete() {
- file.delete();
+ /**
+ * Delete the file
+ *
+ * @return true if file was deleted, false, otherwise
+ */
+ public boolean delete() {
+ return file.delete();
+ }
+
+ /**
+ * @return the relative path
+ */
+ public String getRelativePath() {
+ return path;
+ }
+
+ /**
+ * @return the absolute path
+ */
+ public String getAbsolutePath() {
+ return file.getAbsolutePath();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java
new file mode 100644
index 0000000..bd4567b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.io;
+
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * A node and a path (Can be relative: inside the IO device or absolute inside or outside IO devices)
+ * Used to identify a file/dir across the cluster.
+ */
+public class FileSplit implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private final String node;
+ private final String path;
+ private final boolean managed;
+
+ /**
+ * Constructor
+ *
+ * @param node
+ * @param path
+ * @param maanged
+ */
+ public FileSplit(String node, String path, boolean relative) {
+ this.node = node;
+ this.path = path;
+ this.managed = relative;
+ }
+
+ @Override
+ public String toString() {
+ return "Node: " + node + " " + (managed ? "managed" : "absolute") + " path: " + path;
+ }
+
+ /**
+ * @return the path
+ */
+ public String getPath() {
+ return path;
+ }
+
+ /**
+ * @return true if relative
+ */
+ public boolean isManaged() {
+ return managed;
+ }
+
+ /**
+ * Get the local file represented by this split
+ *
+ * @param ioManager
+ * @return
+ * @throws HyracksDataException
+ */
+ public File getFile(IIOManager ioManager) throws HyracksDataException {
+ return managed ? getFileRef(ioManager).getFile() : new File(path);
+ }
+
+ /**
+ * Get the file reference for the split
+ *
+ * @param ioManager
+ * @return
+ * @throws HyracksDataException
+ */
+ public FileReference getFileRef(IIOManager ioManager) throws HyracksDataException {
+ return ioManager.getFileRef(path, managed);
+ }
+
+ /**
+ * @return the node
+ */
+ public String getNodeName() {
+ return node;
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileDeviceComputer.java
similarity index 62%
rename from asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
rename to hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileDeviceComputer.java
index 6af7441..e75efd4 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileDeviceComputer.java
@@ -16,18 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.api;
+package org.apache.hyracks.api.io;
-import java.io.Serializable;
-
-import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
-public interface ILocalResourceMetadata extends Serializable {
+/**
+ * Computes the device destination for a file from its relative path
+ */
+@FunctionalInterface
+public interface IFileDeviceComputer {
- public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) throws HyracksDataException;
+ /**
+ * Compute the device from the relative path
+ *
+ * @param relativePath
+ * @return
+ */
+ IODeviceHandle compute(String relativePath) throws HyracksDataException;
- public int getDatasetID();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index 63709ed..0b3f71a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -66,5 +66,15 @@
* @param relativePath
* @return A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code relativePath}
*/
- public FileReference getAbsoluteFileRef(int ioDeviceId, String relativePath);
+ public FileReference getFileRef(int ioDeviceId, String relativePath);
+
+ /**
+ * A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code relativePath}
+ *
+ * @param path
+ * @param relative
+ * @return
+ * @throws HyracksDataException
+ */
+ public FileReference getFileRef(String path, boolean relative) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
index 1fcff90..552fbeb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
@@ -20,32 +20,114 @@
import java.io.File;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Represents an IO device
+ */
public class IODeviceHandle implements Serializable {
private static final long serialVersionUID = 1L;
+ /**
+ * Absolute mount point
+ */
+ private final File mount;
+ /**
+ * relative workspace
+ */
+ private final String workspace;
- private final File path;
-
- private final String workAreaPath;
-
- public IODeviceHandle(File path, String workAreaPath) {
- this.path = path;
- workAreaPath = workAreaPath.trim();
- if (workAreaPath.endsWith(File.separator)) {
- workAreaPath = workAreaPath.substring(0, workAreaPath.length() - 1);
- }
- this.workAreaPath = workAreaPath;
+ /**
+ * @param mount
+ * The device root
+ * @param workspace
+ * The relative workspace inside the device
+ */
+ public IODeviceHandle(File mount, String workspace) {
+ this.mount = mount;
+ this.workspace = workspace == null ? null
+ : workspace.endsWith(File.separator) ? workspace.substring(0, workspace.length() - 1)
+ : workspace;
}
- public File getPath() {
- return path;
+ public File getMount() {
+ return mount;
}
- public String getWorkAreaPath() {
- return workAreaPath;
+ public String getWorkspace() {
+ return workspace;
}
- public FileReference createFileReference(String relPath) {
+ /**
+ * Create a file reference
+ *
+ * @param relPath
+ * the relative path
+ * @return
+ */
+ public FileReference createFileRef(String relPath) {
return new FileReference(this, relPath);
}
+
+ /**
+ * Get handles for IO devices
+ *
+ * @param ioDevices
+ * comma separated list of devices
+ * @return
+ */
+ public static List<IODeviceHandle> getDevices(String ioDevices) {
+ List<IODeviceHandle> devices = new ArrayList<>();
+ StringTokenizer tok = new StringTokenizer(ioDevices, ",");
+ while (tok.hasMoreElements()) {
+ String devPath = tok.nextToken().trim();
+ devices.add(new IODeviceHandle(new File(devPath), "."));
+ }
+ return devices;
+ }
+
+ /**
+ * @param absolutePath
+ * @return the relative path
+ * @throws HyracksDataException
+ */
+ public String getRelativePath(String absolutePath) throws HyracksDataException {
+ if (absolutePath.indexOf(mount.getAbsolutePath()) != 0) {
+ throw new HyracksDataException(
+ "Passed path: " + absolutePath + " is not inside the device " + mount.getAbsolutePath());
+ }
+ return absolutePath.substring(mount.getAbsolutePath().length());
+ }
+
+ /**
+ * determinea if the device contains a file with the passed relative path
+ * @param relPath
+ * @return true if it contains, false, otherwise
+ */
+ public boolean contains(String relPath) {
+ return new File(mount, relPath).exists();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ } else if (o instanceof IODeviceHandle) {
+ return mount.getAbsolutePath().equals(((IODeviceHandle) o).getMount().getAbsolutePath());
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return mount.getAbsolutePath().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "mount: " + mount.getAbsolutePath() + ((workspace == null) ? "" : ", workspace: " + workspace);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/DefaultDeviceComputer.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/DefaultDeviceComputer.java
new file mode 100644
index 0000000..bacb608
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/DefaultDeviceComputer.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.io;
+
+import java.io.File;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IFileDeviceComputer;
+import org.apache.hyracks.api.io.IODeviceHandle;
+
+public class DefaultDeviceComputer implements IFileDeviceComputer {
+ private final IOManager ioManager;
+ private AtomicInteger next = new AtomicInteger(0);
+
+ public DefaultDeviceComputer(IOManager ioManager) {
+ this.ioManager = ioManager;
+ }
+
+ @Override
+ public IODeviceHandle compute(String relPath) throws HyracksDataException {
+ String path = relPath;
+ // if number of devices is 1, we return the device
+ if (ioManager.getIODevices().size() == 1) {
+ return ioManager.getIODevices().get(0);
+ }
+ // check if it exists already on a device
+ int nextSeparator = path.lastIndexOf(File.separator);
+ while (nextSeparator > 0) {
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
+ if (dev.contains(path)) {
+ return dev;
+ }
+ }
+ path = path.substring(0, nextSeparator);
+ nextSeparator = path.lastIndexOf(File.separator);
+ }
+ // one last attempt
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
+ if (dev.contains(path)) {
+ return dev;
+ }
+ }
+ // not on any device, round robin assignment
+ return ioManager.getIODevices().get(next.getAndIncrement() % ioManager.getIODevices().size());
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 6588f3b..bdb283a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -31,17 +31,29 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IFileDeviceComputer;
import org.apache.hyracks.api.io.IFileHandle;
import org.apache.hyracks.api.io.IIOFuture;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IODeviceHandle;
public class IOManager implements IIOManager {
+ /*
+ * Constants
+ */
private static final String WORKSPACE_FILE_SUFFIX = ".waf";
+ private static final FilenameFilter WORKSPACE_FILES_FILTER = (dir, name) -> name.endsWith(WORKSPACE_FILE_SUFFIX);
+ /*
+ * Finals
+ */
private final List<IODeviceHandle> ioDevices;
+ private final List<IODeviceHandle> workspaces;
+ /*
+ * Mutables
+ */
private Executor executor;
- private final List<IODeviceHandle> workAreaIODevices;
- private int workAreaDeviceIndex;
+ private int workspaceIndex;
+ private IFileDeviceComputer deviceComputer;
public IOManager(List<IODeviceHandle> devices, Executor executor) throws HyracksException {
this(devices);
@@ -50,19 +62,21 @@
public IOManager(List<IODeviceHandle> devices) throws HyracksException {
this.ioDevices = Collections.unmodifiableList(devices);
- workAreaIODevices = new ArrayList<>();
+ workspaces = new ArrayList<>();
for (IODeviceHandle d : ioDevices) {
- if (d.getWorkAreaPath() != null) {
- new File(d.getPath(), d.getWorkAreaPath()).mkdirs();
- workAreaIODevices.add(d);
+ if (d.getWorkspace() != null) {
+ new File(d.getMount(), d.getWorkspace()).mkdirs();
+ workspaces.add(d);
}
}
- if (workAreaIODevices.isEmpty()) {
+ if (workspaces.isEmpty()) {
throw new HyracksException("No devices with work areas found");
}
- workAreaDeviceIndex = 0;
+ workspaceIndex = 0;
+ deviceComputer = new DefaultDeviceComputer(this);
}
+ @Override
public void setExecutor(Executor executor) {
this.executor = executor;
}
@@ -200,16 +214,16 @@
}
public synchronized FileReference createWorkspaceFile(String prefix) throws HyracksDataException {
- IODeviceHandle dev = workAreaIODevices.get(workAreaDeviceIndex);
- workAreaDeviceIndex = (workAreaDeviceIndex + 1) % workAreaIODevices.size();
- String waPath = dev.getWorkAreaPath();
+ IODeviceHandle dev = workspaces.get(workspaceIndex);
+ workspaceIndex = (workspaceIndex + 1) % workspaces.size();
+ String waPath = dev.getWorkspace();
File waf;
try {
- waf = File.createTempFile(prefix, WORKSPACE_FILE_SUFFIX, new File(dev.getPath(), waPath));
+ waf = File.createTempFile(prefix, WORKSPACE_FILE_SUFFIX, new File(dev.getMount(), waPath));
} catch (IOException e) {
throw new HyracksDataException(e);
}
- return dev.createFileReference(waPath + File.separator + waf.getName());
+ return dev.createFileRef(waPath + File.separator + waf.getName());
}
private abstract class AsyncRequest implements IIOFuture, Runnable {
@@ -302,8 +316,8 @@
@Override
public void deleteWorkspaceFiles() {
- for (IODeviceHandle ioDevice : workAreaIODevices) {
- File workspaceFolder = new File(ioDevice.getPath(), ioDevice.getWorkAreaPath());
+ for (IODeviceHandle ioDevice : workspaces) {
+ File workspaceFolder = new File(ioDevice.getMount(), ioDevice.getWorkspace());
if (workspaceFolder.exists() && workspaceFolder.isDirectory()) {
File[] workspaceFiles = workspaceFolder.listFiles(WORKSPACE_FILES_FILTER);
for (File workspaceFile : workspaceFiles) {
@@ -313,15 +327,31 @@
}
}
- private static final FilenameFilter WORKSPACE_FILES_FILTER = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.endsWith(WORKSPACE_FILE_SUFFIX);
- }
- };
-
@Override
- public FileReference getAbsoluteFileRef(int ioDeviceId, String relativePath) {
+ public FileReference getFileRef(int ioDeviceId, String relativePath) {
IODeviceHandle devHandle = ioDevices.get(ioDeviceId);
return new FileReference(devHandle, relativePath);
}
+
+ @Override
+ public FileReference getFileRef(String path, boolean relative) throws HyracksDataException {
+ if (relative) {
+ return new FileReference(deviceComputer.compute(path), path);
+ }
+ IODeviceHandle devHandle = getDevice(path);
+ if (devHandle == null) {
+ throw new HyracksDataException("The file with absolute path: " + path + " is outside all io devices");
+ }
+ String relativePath = devHandle.getRelativePath(path);
+ return new FileReference(devHandle, relativePath);
+ }
+
+ public IODeviceHandle getDevice(String fullPath) {
+ for (IODeviceHandle d : ioDevices) {
+ if (fullPath.startsWith(d.getMount().getAbsolutePath())) {
+ return d;
+ }
+ }
+ return null;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
index 189a3a1..6703b0f 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
@@ -69,7 +70,7 @@
RecordDescriptor desc = recordDescriptors[0];
IRecordReader reader;
try {
- reader = createRecordReader(split.getLocalFile().getFile(), desc);
+ reader = createRecordReader(split.getFile(null), desc);
} catch (Exception e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
index 6b0ee4d..5028ccc 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java
index 028a91a..abd8d2c 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.dataflow.std.file;
+import org.apache.hyracks.api.io.FileSplit;
+
public class ConstantFileSplitProvider implements IFileSplitProvider {
private static final long serialVersionUID = 1L;
private final FileSplit[] splits;
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java
index c3883e8..94c5a68 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
@@ -61,8 +62,6 @@
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
final FileSplit split = fileSplitProvider.getFileSplits()[partition];
- final String path = split.getLocalFile().getFile().getPath();
- final int deviceId = split.getIODeviceId();
final IIOManager ioManager = ctx.getIOManager();
return new AbstractOperatorNodePushable() {
@@ -73,7 +72,8 @@
@Override
public void initialize() throws HyracksDataException {
- File f = ioManager.getAbsoluteFileRef(deviceId, path).getFile();
+ // will only work for files inside the io devices
+ File f = ioManager.getFileRef(split.getPath(), split.isManaged()).getFile();
if (quietly) {
FileUtils.deleteQuietly(f);
} else {
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
index 5a3d776..311f79e 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
@@ -28,6 +28,8 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -52,10 +54,11 @@
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
final FileSplit split = fileSplitProvider.getFileSplits()[partition];
final ITupleParser tp = tupleParserFactory.createTupleParser(ctx);
+ final IIOManager ioManager = ctx.getIOManager();
return new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
- File f = split.getLocalFile().getFile();
+ File f = split.getFile(ioManager);
try {
writer.open();
InputStream in;
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileSplit.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileSplit.java
deleted file mode 100644
index b201acd..0000000
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileSplit.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.dataflow.std.file;
-
-import java.io.File;
-import java.io.Serializable;
-
-import org.apache.hyracks.api.io.FileReference;
-
-public class FileSplit implements Serializable {
- private static final long serialVersionUID = 1L;
- private final FileReference file;
- private final int ioDeviceId;
- private final int partition;
- private final String nodeName;
-
- public FileSplit(String nodeName, FileReference file) {
- this.nodeName = nodeName;
- this.file = file;
- this.ioDeviceId = 0;
- this.partition = -1;
- }
-
- public FileSplit(String nodeName, FileReference file, int ioDeviceId, int partition) {
- this.nodeName = nodeName;
- this.file = file;
- this.ioDeviceId = ioDeviceId;
- this.partition = partition;
- }
-
- public FileSplit(String nodeName, String path, int ioDeviceId) {
- this.nodeName = nodeName;
- this.file = new FileReference(new File(path));
- this.ioDeviceId = ioDeviceId;
- this.partition = -1;
- }
-
- public FileSplit(String nodeName, String path) {
- this.nodeName = nodeName;
- this.file = new FileReference(new File(path));
- this.ioDeviceId = 0;
- this.partition = -1;
- }
-
- public String getNodeName() {
- return nodeName;
- }
-
- public FileReference getLocalFile() {
- return file;
- }
-
- public int getIODeviceId() {
- return ioDeviceId;
- }
-
- public int getPartition() {
- return partition;
- }
-
- @Override
- public String toString() {
- return file.toString();
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
index ae9585e..e398bc0 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
@@ -28,6 +28,8 @@
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
@@ -46,13 +48,14 @@
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
final FileSplit[] splits = fileSplitProvider.getFileSplits();
+ final IIOManager ioManager = ctx.getIOManager();
return new AbstractUnaryInputSinkOperatorNodePushable() {
private OutputStream out;
@Override
public void open() throws HyracksDataException {
try {
- out = new FileOutputStream(splits[partition].getLocalFile().getFile());
+ out = new FileOutputStream(splits[partition].getFile(ioManager));
} catch (FileNotFoundException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java
index cf2e923..a85e76e 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java
@@ -20,6 +20,8 @@
import java.io.Serializable;
+import org.apache.hyracks.api.io.FileSplit;
+
public interface IFileSplitProvider extends Serializable {
public FileSplit[] getFileSplits();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
index 1a19a2b..1186283 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
@@ -22,6 +22,7 @@
import java.io.FileOutputStream;
import java.io.OutputStream;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
public class LineFileWriteOperatorDescriptor extends AbstractFileWriteOperatorDescriptor {
@@ -59,6 +60,6 @@
@Override
protected IRecordWriter createRecordWriter(FileSplit fileSplit, int index) throws Exception {
- return new LineWriterImpl(fileSplit.getLocalFile().getFile(), columns, separator);
+ return new LineWriterImpl(fileSplit.getFile(null), columns, separator);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
index 33ac540..8df7fa7 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
@@ -29,6 +29,8 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
@@ -72,6 +74,7 @@
throws HyracksDataException {
// Output files
final FileSplit[] splits = fileSplitProvider.getFileSplits();
+ IIOManager ioManager = ctx.getIOManager();
// Frame accessor
final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(
recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
@@ -87,7 +90,7 @@
@Override
public void open() throws HyracksDataException {
try {
- out = new BufferedWriter(new FileWriter(splits[partition].getLocalFile().getFile()));
+ out = new BufferedWriter(new FileWriter(splits[partition].getFile(ioManager)));
bbis = new ByteBufferInputStream();
di = new DataInputStream(bbis);
} catch (Exception e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
index e807301..97f12b5 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
@@ -25,6 +25,7 @@
import java.io.IOException;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
public class RecordFileScanOperatorDescriptor extends AbstractDeserializedFileScanOperatorDescriptor {
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
index 16d2158..da5b4a8 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
@@ -48,6 +48,7 @@
*/
public class SortGroupByOperatorDescriptor extends AbstractSorterOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
private final int[] groupFields;
private final IAggregatorDescriptorFactory mergeAggregatorFactory;
private final IAggregatorDescriptorFactory partialAggregatorFactory;
@@ -102,6 +103,8 @@
@Override
public AbstractSorterOperatorDescriptor.SortActivity getSortActivity(ActivityId id) {
return new AbstractSorterOperatorDescriptor.SortActivity(id) {
+ private static final long serialVersionUID = 1L;
+
@Override
protected AbstractSortRunGenerator getRunGenerator(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescriptorProvider) throws HyracksDataException {
@@ -116,6 +119,7 @@
@Override
public AbstractSorterOperatorDescriptor.MergeActivity getMergeActivity(ActivityId id) {
return new AbstractSorterOperatorDescriptor.MergeActivity(id) {
+ private static final long serialVersionUID = 1L;
@Override
protected AbstractExternalSortRunMerger getSortRunMerger(IHyracksTaskContext ctx,
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java
index 98807eb..a213ca1 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java
@@ -51,6 +51,7 @@
*/
public class IntersectOperatorDescriptor extends AbstractOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
private final int[][] projectFields;
private final INormalizedKeyComputerFactory firstKeyNormalizerFactory;
private final IBinaryComparatorFactory[] comparatorFactory;
@@ -110,6 +111,8 @@
private class IntersectActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
public IntersectActivity(ActivityId activityId) {
super(activityId);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
index 36d0c1b..8597ed6 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
@@ -64,6 +64,8 @@
@Override
public AbstractSorterOperatorDescriptor.SortActivity getSortActivity(ActivityId id) {
return new AbstractSorterOperatorDescriptor.SortActivity(id) {
+ private static final long serialVersionUID = 1L;
+
@Override
protected AbstractSortRunGenerator getRunGenerator(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider) throws HyracksDataException {
@@ -76,6 +78,8 @@
@Override
public AbstractSorterOperatorDescriptor.MergeActivity getMergeActivity(ActivityId id) {
return new AbstractSorterOperatorDescriptor.MergeActivity(id) {
+ private static final long serialVersionUID = 1L;
+
@Override
protected ExternalSortRunMerger getSortRunMerger(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, IFrameWriter writer, ISorter sorter,
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
index 5df09c6..218faaf 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
@@ -35,6 +35,7 @@
public class TopKSorterOperatorDescriptor extends AbstractSorterOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
private final int topK;
public TopKSorterOperatorDescriptor(IOperatorDescriptorRegistry spec, int framesLimit, int topK, int[] sortFields,
@@ -47,6 +48,8 @@
@Override
public SortActivity getSortActivity(ActivityId id) {
return new SortActivity(id) {
+ private static final long serialVersionUID = 1L;
+
@Override
protected AbstractSortRunGenerator getRunGenerator(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider) {
@@ -60,6 +63,8 @@
@Override
public MergeActivity getMergeActivity(ActivityId id) {
return new MergeActivity(id) {
+ private static final long serialVersionUID = 1L;
+
@Override
protected ExternalSortRunMerger getSortRunMerger(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, IFrameWriter writer, ISorter sorter,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java
index aa386a8..c6e81a4 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -19,9 +19,6 @@
package org.apache.hyracks.examples.btree.client;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
@@ -54,6 +51,8 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
// This example will insert tuples into the primary and secondary index using an insert pipeline
@@ -79,6 +78,9 @@
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
+
+ @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+ public boolean relative = true;
}
public static void main(String[] args) throws Exception {
@@ -148,7 +150,8 @@
int[] primaryFieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input
// tuple to field 0 of
// B-Tree tuple, etc.
- IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
+ IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName,
+ options.relative);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
@@ -175,7 +178,7 @@
// tuple
int[] secondaryFieldPermutation = { 1, 2 };
IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs,
- options.secondaryBTreeName);
+ options.secondaryBTreeName, true);
// create operator descriptor
TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
spec, recDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java
index a189b40..05143a6 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java
@@ -19,25 +19,22 @@
package org.apache.hyracks.examples.btree.client;
-import java.io.File;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
public class JobHelper {
- public static IFileSplitProvider createFileSplitProvider(String[] splitNCs, String btreeFileName) {
+ public static IFileSplitProvider createFileSplitProvider(String[] splitNCs, String btreeFileName,
+ boolean relative) {
FileSplit[] fileSplits = new FileSplit[splitNCs.length];
for (int i = 0; i < splitNCs.length; ++i) {
String fileName = btreeFileName + "." + splitNCs[i];
- fileSplits[i] = new FileSplit(splitNCs[i], new FileReference(new File(fileName)));
+ fileSplits[i] = new FileSplit(splitNCs[i], fileName, relative);
}
- IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
- return splitProvider;
+ return new ConstantFileSplitProvider(fileSplits);
}
public static void createPartitionConstraint(JobSpecification spec, IOperatorDescriptor op, String[] splitNCs) {
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index eaf9109..c455c0d 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -18,9 +18,6 @@
*/
package org.apache.hyracks.examples.btree.client;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
@@ -52,6 +49,8 @@
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
// This example will load a primary index from randomly generated data
@@ -77,6 +76,9 @@
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
+
+ @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+ public boolean relative = true;
}
public static void main(String[] args) throws Exception {
@@ -152,7 +154,8 @@
int[] fieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input tuple
// to field 0 of B-Tree tuple,
// etc.
- IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
+ IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName,
+ options.relative);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, recDesc,
storageManager, lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index 6b22681..1867a80 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -20,9 +20,6 @@
import java.io.DataOutput;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -50,6 +47,8 @@
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
// This example will perform an ordered scan on the primary index
// i.e. a range-search for [-infinity, +infinity]
@@ -70,6 +69,9 @@
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
+
+ @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+ public boolean relative = true;
}
public static void main(String[] args) throws Exception {
@@ -142,7 +144,8 @@
int[] highKeyFields = { 1 }; // low key is in field 1 of tuples going
// into search op
- IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
+ IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName,
+ options.relative);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager,
lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null, lowKeyFields,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index 9b85b8b..2acc204 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -18,9 +18,6 @@
*/
package org.apache.hyracks.examples.btree.client;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -47,6 +44,8 @@
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexDiskOrderScanOperatorDescriptor;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
// This example will load a secondary index with <key, primary-index key> pairs
// We require an existing primary index built with PrimaryIndexBulkLoadExample
@@ -73,6 +72,9 @@
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
+
+ @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+ public boolean relative = true;
}
public static void main(String[] args) throws Exception {
@@ -123,7 +125,8 @@
comparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// use a disk-order scan to read primary index
- IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
+ IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName,
+ options.relative);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
TreeIndexDiskOrderScanOperatorDescriptor btreeScanOp = new TreeIndexDiskOrderScanOperatorDescriptor(spec,
recDesc, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits,
@@ -146,7 +149,8 @@
// the B-Tree expects its keyfields to be at the front of its input
// tuple
int[] fieldPermutation = { 1, 0 };
- IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName);
+ IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName,
+ options.relative);
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, null,
storageManager, lcManagerProvider, btreeSplitProvider, secondaryTypeTraits, comparatorFactories, null,
fieldPermutation, 0.7f, false, 1000L, true, dataflowHelperFactory);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index 94ae0d3..5e140ce 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -20,9 +20,6 @@
import java.io.DataOutput;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -50,6 +47,8 @@
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
// This example will perform range search on the secondary index
// and then retrieve the corresponding source records from the primary index
@@ -73,6 +72,9 @@
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
+
+ @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+ public boolean relative = true;
}
public static void main(String[] args) throws Exception {
@@ -168,7 +170,7 @@
// index search op
IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs,
- options.secondaryBTreeName);
+ options.secondaryBTreeName, options.relative);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
BTreeSearchOperatorDescriptor secondarySearchOp = new BTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
@@ -186,7 +188,8 @@
// going into primary index search
// op
- IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
+ IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName,
+ options.relative);
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
null, primaryLowKeyFields, primaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java
index 837a8a4..c2b5388 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java
@@ -22,16 +22,13 @@
import java.io.DataOutput;
import java.io.File;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
@@ -43,7 +40,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
@@ -64,6 +60,8 @@
import org.apache.hyracks.test.support.TestStorageManagerInterface;
import org.apache.hyracks.tests.am.common.ITreeIndexOperatorTestHelper;
import org.apache.hyracks.tests.integration.AbstractIntegrationTest;
+import org.junit.After;
+import org.junit.Before;
public abstract class AbstractBTreeOperatorTest extends AbstractIntegrationTest {
static {
@@ -88,6 +86,7 @@
// to be set by subclasses
protected String primaryFileName;
+ protected boolean primaryRelative;
protected IFileSplitProvider primarySplitProvider;
// field, type and key declarations for secondary indexes
@@ -101,6 +100,7 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
protected String secondaryFileName;
+ protected boolean secondaryRelative;
protected IFileSplitProvider secondarySplitProvider;
protected ITreeIndexOperatorTestHelper testHelper;
@@ -114,11 +114,13 @@
testHelper = createTestHelper();
dataflowHelperFactory = createDataFlowHelperFactory();
primaryFileName = testHelper.getPrimaryIndexName();
- primarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID, new FileReference(
- new File(primaryFileName))) });
+ primaryRelative = testHelper.getPrimaryIndexNameRelative();
+ primarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID, primaryFileName,
+ primaryRelative) });
secondaryFileName = testHelper.getSecondaryIndexName();
+ secondaryRelative = testHelper.getSecondaryIndexNameRelative();
secondarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File(secondaryFileName))) });
+ secondaryFileName, secondaryRelative) });
// field, type and key declarations for primary index
primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
@@ -156,8 +158,8 @@
protected void loadPrimaryIndex() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/orders-part1.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -268,8 +270,8 @@
IndexOperation pipelineOperation = useUpsert ? IndexOperation.UPSERT : IndexOperation.INSERT;
JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/orders-part2.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java
index d4fb56d..0bb02f8 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java
@@ -21,28 +21,28 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.junit.Before;
+import org.junit.Test;
public class BTreePrimaryIndexScanOperatorTest extends AbstractBTreeOperatorTest {
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -80,7 +80,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java
index e690423..baca105 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java
@@ -21,28 +21,28 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.junit.Before;
+import org.junit.Test;
public class BTreePrimaryIndexSearchOperatorTest extends AbstractBTreeOperatorTest {
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -85,7 +85,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexStatsOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexStatsOperatorTest.java
index 3791896..911783c 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexStatsOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreePrimaryIndexStatsOperatorTest.java
@@ -19,23 +19,23 @@
package org.apache.hyracks.tests.am.btree;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexStatsOperatorDescriptor;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.junit.Before;
+import org.junit.Test;
public class BTreePrimaryIndexStatsOperatorTest extends AbstractBTreeOperatorTest {
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -52,7 +52,7 @@
primaryBloomFilterKeyFields, dataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryStatsOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java
index 6ebc177..27c15ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java
@@ -21,28 +21,28 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.junit.Before;
+import org.junit.Test;
public class BTreeSecondaryIndexInsertOperatorTest extends AbstractBTreeOperatorTest {
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -103,7 +103,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java
index 2bba010..5996ca0 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java
@@ -21,28 +21,28 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.junit.Before;
+import org.junit.Test;
public class BTreeSecondaryIndexSearchOperatorTest extends AbstractBTreeOperatorTest {
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -103,7 +103,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java
index 031ef76..79fcae3 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java
@@ -20,28 +20,28 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.junit.Before;
+import org.junit.Test;
public class BTreeSecondaryIndexUpsertOperatorTest extends AbstractBTreeOperatorTest {
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -102,7 +102,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/ITreeIndexOperatorTestHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/ITreeIndexOperatorTestHelper.java
index bd9bcea..9f76706 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/ITreeIndexOperatorTestHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/ITreeIndexOperatorTestHelper.java
@@ -22,7 +22,11 @@
public interface ITreeIndexOperatorTestHelper {
public String getPrimaryIndexName();
+ public boolean getPrimaryIndexNameRelative();
+
public String getSecondaryIndexName();
+ public boolean getSecondaryIndexNameRelative();
+
public void cleanup(String primaryFileName, String secondaryFileName);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/LSMTreeOperatorTestHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/LSMTreeOperatorTestHelper.java
index 3d11dae..6e82f4f 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/LSMTreeOperatorTestHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/LSMTreeOperatorTestHelper.java
@@ -41,20 +41,32 @@
DEFAULT_MEM_NUM_PAGES);
}
+ @Override
public String getPrimaryIndexName() {
return "primary" + simpleDateFormat.format(new Date());
}
+ @Override
+ public boolean getPrimaryIndexNameRelative() {
+ return true;
+ }
+
+ @Override
public String getSecondaryIndexName() {
return "secondary" + simpleDateFormat.format(new Date());
}
@Override
+ public boolean getSecondaryIndexNameRelative() {
+ return true;
+ }
+
+ @Override
public void cleanup(String primaryFileName, String secondaryFileName) {
for (IODeviceHandle dev : ioManager.getIODevices()) {
- File primaryDir = new File(dev.getPath(), primaryFileName);
+ File primaryDir = new File(dev.getMount(), primaryFileName);
cleanupDir(primaryDir);
- File secondaryDir = new File(dev.getPath(), secondaryFileName);
+ File secondaryDir = new File(dev.getMount(), secondaryFileName);
cleanupDir(secondaryDir);
}
}
@@ -64,6 +76,7 @@
return;
}
FilenameFilter filter = new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/TreeOperatorTestHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/TreeOperatorTestHelper.java
index a40a96b..dd03568 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/TreeOperatorTestHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/common/TreeOperatorTestHelper.java
@@ -28,12 +28,14 @@
protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final String sep = System.getProperty("file.separator");
+ @Override
public String getPrimaryIndexName() {
- return System.getProperty("java.io.tmpdir") + sep + "primary" + simpleDateFormat.format(new Date());
+ return System.getProperty("java.io.tmpdir") + "primary" + simpleDateFormat.format(new Date());
}
+ @Override
public String getSecondaryIndexName() {
- return System.getProperty("java.io.tmpdir") + sep + "secondary" + simpleDateFormat.format(new Date());
+ return System.getProperty("java.io.tmpdir") + "secondary" + simpleDateFormat.format(new Date());
}
@Override
@@ -47,4 +49,14 @@
secondary.deleteOnExit();
}
}
+
+ @Override
+ public boolean getPrimaryIndexNameRelative() {
+ return false;
+ }
+
+ @Override
+ public boolean getSecondaryIndexNameRelative() {
+ return false;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
index d13cc19..b6e84e4 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
@@ -22,9 +22,6 @@
import java.io.DataOutput;
import java.io.File;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -32,7 +29,7 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.DoublePointable;
@@ -47,7 +44,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
@@ -74,6 +70,8 @@
import org.apache.hyracks.test.support.TestStorageManagerInterface;
import org.apache.hyracks.tests.am.common.ITreeIndexOperatorTestHelper;
import org.apache.hyracks.tests.integration.AbstractIntegrationTest;
+import org.junit.After;
+import org.junit.Before;
public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest {
static {
@@ -108,6 +106,7 @@
// to be set by subclasses
protected String primaryFileName;
+ protected boolean primaryFileNameRelative;
protected IFileSplitProvider primarySplitProvider;
// field, type and key declarations for secondary indexes
@@ -127,6 +126,7 @@
protected IBinaryComparatorFactory[] btreeComparatorFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
protected String secondaryFileName;
+ protected boolean secondaryFileNameRelative;
protected IFileSplitProvider secondarySplitProvider;
protected ITreeIndexOperatorTestHelper testHelper;
@@ -140,11 +140,13 @@
testHelper = createTestHelper();
primaryFileName = testHelper.getPrimaryIndexName();
- primarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID, new FileReference(
- new File(primaryFileName))) });
+ primaryFileNameRelative = testHelper.getPrimaryIndexNameRelative();
+ primarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID, primaryFileName,
+ primaryFileNameRelative) });
secondaryFileName = testHelper.getSecondaryIndexName();
+ secondaryFileNameRelative = testHelper.getSecondaryIndexNameRelative();
secondarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File(secondaryFileName))) });
+ secondaryFileName, secondaryFileNameRelative) });
// field, type and key declarations for primary index
primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
@@ -216,8 +218,8 @@
protected void loadPrimaryIndex() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/orders-with-locations-part1.txt"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/orders-with-locations-part1.txt").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -327,8 +329,8 @@
JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/orders-with-locations-part2.txt"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/orders-with-locations-part2.txt").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
index df22fe5..439ff23 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
@@ -21,21 +21,18 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
@@ -46,6 +43,8 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.Before;
+import org.junit.Test;
public class RTreeSecondaryIndexInsertOperatorTest extends AbstractRTreeOperatorTest {
@@ -114,7 +113,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
index 379c3cb..d190cfe 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
@@ -21,21 +21,18 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
@@ -44,6 +41,8 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.Before;
+import org.junit.Test;
public class RTreeSecondaryIndexScanOperatorTest extends AbstractRTreeOperatorTest {
@@ -97,7 +96,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
index 8511443..8426c95 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
@@ -21,21 +21,18 @@
import java.io.DataOutput;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
@@ -46,6 +43,8 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.Before;
+import org.junit.Test;
public class RTreeSecondaryIndexSearchOperatorTest extends AbstractRTreeOperatorTest {
public RTreeSecondaryIndexSearchOperatorTest() {
@@ -110,7 +109,7 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
index 863d8c2..200fde0 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
@@ -19,17 +19,14 @@
package org.apache.hyracks.tests.am.rtree;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -37,6 +34,8 @@
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexStatsOperatorDescriptor;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.Before;
+import org.junit.Test;
public class RTreeSecondaryIndexStatsOperatorTest extends AbstractRTreeOperatorTest {
public RTreeSecondaryIndexStatsOperatorTest() {
@@ -62,7 +61,7 @@
rtreeDataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryStatsOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- createTempFile().getAbsolutePath()) });
+ createTempFile().getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java
index e3093c4..a1bc086 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java
@@ -29,7 +29,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -50,7 +50,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
import org.apache.hyracks.dataflow.std.group.HashSpillableTableFactory;
@@ -74,7 +73,8 @@
public class AggregationTest extends AbstractIntegrationTest {
final IFileSplitProvider splitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/lineitem.tbl"))) });
+ new FileSplit[] { new FileSplit(NC2_ID, new File("data/tpch0.001/lineitem.tbl").getAbsolutePath(),
+ false) });
final RecordDescriptor desc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), IntegerSerializerDeserializer.INSTANCE,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java
index d12d534..adcb89b 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java
@@ -20,8 +20,6 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -30,7 +28,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -41,13 +39,12 @@
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import org.apache.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import org.apache.hyracks.dataflow.std.connectors.MToNBroadcastConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
import org.apache.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
@@ -57,13 +54,15 @@
import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import org.apache.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
+import org.junit.Test;
public class CountOfCountsTest extends AbstractIntegrationTest {
@Test
public void countOfCountsSingleNC() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File("data/words.txt"))) };
+ FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new File("data/words.txt").getAbsolutePath(),
+ false) };
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
RecordDescriptor desc = new RecordDescriptor(
new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
@@ -135,7 +134,8 @@
public void countOfCountsMultiNC() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File("data/words.txt"))) };
+ FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new File("data/words.txt").getAbsolutePath(),
+ false) };
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
RecordDescriptor desc = new RecordDescriptor(
new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
@@ -208,7 +208,8 @@
public void countOfCountsExternalSortMultiNC() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File("data/words.txt"))) };
+ FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new File("data/words.txt").getAbsolutePath(),
+ false) };
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
RecordDescriptor desc = new RecordDescriptor(
new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java
index 699389b..1edf3e9 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java
@@ -20,8 +20,6 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -29,7 +27,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -44,13 +42,13 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.LimitOperatorDescriptor;
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.sort.TopKSorterOperatorDescriptor;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
+import org.junit.Test;
public class HeapSortMergeTest extends AbstractIntegrationTest {
@@ -59,8 +57,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -90,7 +88,7 @@
File file = File.createTempFile(getClass().getName(), ".tmp");
IFileSplitProvider outputSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, file.getAbsolutePath()) });
+ new FileSplit[] { new FileSplit(NC1_ID, file.getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outputSplitProvider, "|");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
@@ -116,8 +114,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
index b76e458..30e2e03 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
@@ -32,7 +32,7 @@
@Test
public void failureOnCreatePushRuntime() throws Exception {
- for (int round = 0; round < 1000; ++round) {
+ for (int round = 0; round < 100; ++round) {
execTest();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java
index ad3c006..de65776 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java
@@ -30,7 +30,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -54,7 +54,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
import org.apache.hyracks.dataflow.std.group.HashSpillableTableFactory;
@@ -70,10 +69,11 @@
public class LocalityAwareConnectorTest extends AbstractMultiNCIntegrationTest {
final IFileSplitProvider splitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit("asterix-001", new FileReference(new File("data/tpch0.001/lineitem.tbl"))),
- new FileSplit("asterix-002", new FileReference(new File("data/tpch0.001/lineitem.tbl"))),
- new FileSplit("asterix-003", new FileReference(new File("data/tpch0.001/lineitem.tbl"))),
- new FileSplit("asterix-004", new FileReference(new File("data/tpch0.001/lineitem.tbl"))) });
+ new FileSplit[] { new FileSplit("asterix-001", new File("data/tpch0.001/lineitem.tbl").getAbsolutePath(),
+ false),
+ new FileSplit("asterix-002", new File("data/tpch0.001/lineitem.tbl").getAbsolutePath(), false),
+ new FileSplit("asterix-003", new File("data/tpch0.001/lineitem.tbl").getAbsolutePath(), false),
+ new FileSplit("asterix-004", new File("data/tpch0.001/lineitem.tbl").getAbsolutePath(), false) });
final int fileSize = 800 * 1024 * 4;
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java
index 9a56cd3..3bb35ad 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
@@ -36,7 +37,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.misc.ReplicateOperatorDescriptor;
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
@@ -72,7 +72,8 @@
outputFile[i].deleteOnExit();
}
- FileSplit[] inputSplits = new FileSplit[] { new FileSplit(NC1_ID, inputFileName) };
+ FileSplit[] inputSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(inputFileName).getAbsolutePath(),
+ false) };
String[] locations = new String[] { NC1_ID };
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java
index 08cc0b8..e3eb98f 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java
@@ -20,8 +20,6 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -29,7 +27,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
@@ -45,10 +43,10 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
+import org.junit.Test;
public class ScanPrintTest extends AbstractIntegrationTest {
@Test
@@ -56,8 +54,8 @@
JobSpecification spec = new JobSpecification();
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(new FileSplit[] {
- new FileSplit(NC2_ID, new FileReference(new File("data/words.txt"))),
- new FileSplit(NC1_ID, new FileReference(new File("data/words.txt"))) });
+ new FileSplit(NC2_ID, new File("data/words.txt").getAbsolutePath(), false),
+ new FileSplit(NC1_ID, new File("data/words.txt").getAbsolutePath(), false) });
RecordDescriptor desc = new RecordDescriptor(
new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
@@ -87,8 +85,8 @@
public void scanPrint02() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -126,8 +124,8 @@
public void scanPrint03() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java
index a31bef7..9f05aac 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java
@@ -20,8 +20,6 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -29,7 +27,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -44,12 +42,12 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import org.apache.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
+import org.junit.Test;
public class SortMergeTest extends AbstractIntegrationTest {
@Test
@@ -57,8 +55,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -105,8 +103,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java
index a4c87c8..7b6f5ea 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java
@@ -21,8 +21,6 @@
import java.io.File;
import java.util.Arrays;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -30,7 +28,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.UTF8StringBinaryHashFunctionFamily;
@@ -42,12 +40,12 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.join.JoinComparatorFactory;
import org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor;
import org.apache.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
+import org.junit.Test;
public class TPCHCustomerOptimizedHybridHashJoinTest extends AbstractIntegrationTest {
@@ -88,7 +86,7 @@
private IOperatorDescriptor getPrinter(JobSpecification spec, File file) {
IFileSplitProvider outputSplitProvider = new ConstantFileSplitProvider(
new FileSplit[] {
- new FileSplit(NC1_ID, file.getAbsolutePath()) });
+ new FileSplit(NC1_ID, file.getAbsolutePath(), false) });
return DEBUG ? new PlainFileWriterOperatorDescriptor(spec, outputSplitProvider, "|")
: new NullSinkOperatorDescriptor(spec);
@@ -97,12 +95,12 @@
@Test
public void customerOrderCIDHybridHashJoin_Case1() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer4.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer4.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders4.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders4.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitsProvider,
@@ -146,12 +144,12 @@
public void customerOrderCIDHybridHashJoin_Case2() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer3.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer3.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders4.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders4.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
@@ -197,12 +195,12 @@
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer3.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer3.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders1.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders1.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
index 43ef74d..007c7b5 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
@@ -20,8 +20,6 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -31,7 +29,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -40,13 +38,12 @@
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import org.apache.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import org.apache.hyracks.dataflow.std.connectors.MToNBroadcastConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.join.GraceHashJoinOperatorDescriptor;
import org.apache.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
@@ -55,6 +52,7 @@
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.tests.util.NoopMissingWriterFactory;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
+import org.junit.Test;
public class TPCHCustomerOrderHashJoinTest extends AbstractIntegrationTest {
@@ -74,8 +72,8 @@
public void customerOrderCIDJoin() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -83,8 +81,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -153,8 +151,8 @@
public void customerOrderCIDGraceJoin() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -162,8 +160,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -236,8 +234,8 @@
public void customerOrderCIDHybridHashJoin() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -245,8 +243,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -319,8 +317,8 @@
public void customerOrderCIDInMemoryHashLeftOuterJoin() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -328,8 +326,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -403,8 +401,8 @@
public void customerOrderCIDGraceHashLeftOuterJoin() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -412,8 +410,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -491,8 +489,8 @@
public void customerOrderCIDHybridHashLeftOuterJoin() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -500,8 +498,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -580,8 +578,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -590,8 +588,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -667,8 +665,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -677,8 +675,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -758,8 +756,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -768,8 +766,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -849,8 +847,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -859,8 +857,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -936,8 +934,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -946,8 +944,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
index 3cd34f3..fac1815 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
@@ -20,8 +20,6 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -36,7 +34,7 @@
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
@@ -48,12 +46,12 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.join.NestedLoopJoinOperatorDescriptor;
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.tests.util.NoopMissingWriterFactory;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
+import org.junit.Test;
public class TPCHCustomerOrderNestedLoopJoinTest extends AbstractIntegrationTest {
private static class JoinComparatorFactory implements ITuplePairComparatorFactory {
@@ -128,8 +126,8 @@
public void customerOrderCIDJoin() throws Exception {
JobSpecification spec = new JobSpecification();
- FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/customer.tbl"))) };
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/customer.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -137,8 +135,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/orders.tbl"))) };
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new File(
+ "data/tpch0.001/orders.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -204,8 +202,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -214,8 +212,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -281,8 +279,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -291,8 +289,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -358,8 +356,8 @@
JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/customer-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/customer-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(custSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -368,8 +366,8 @@
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File("data/tpch0.001/orders-part1.tbl"))),
- new FileSplit(NC2_ID, new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl").getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl").getAbsolutePath(), false) };
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java
index 12e04f0..7824071 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java
@@ -20,14 +20,12 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
@@ -36,11 +34,11 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
import org.apache.hyracks.tests.util.ResultSerializerFactoryProvider;
+import org.junit.Test;
public class UnionTest extends AbstractIntegrationTest {
@Test
@@ -48,8 +46,8 @@
JobSpecification spec = new JobSpecification();
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(new FileSplit[] {
- new FileSplit(NC2_ID, new FileReference(new File("data/words.txt"))),
- new FileSplit(NC1_ID, new FileReference(new File("data/words.txt"))) });
+ new FileSplit(NC2_ID, new File("data/words.txt").getAbsolutePath(), false),
+ new FileSplit(NC1_ID, new File("data/words.txt").getAbsolutePath(), false) });
RecordDescriptor desc = new RecordDescriptor(
new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/VSizeFrameSortMergeTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/VSizeFrameSortMergeTest.java
index 039497c..ecfd212 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/VSizeFrameSortMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/VSizeFrameSortMergeTest.java
@@ -21,15 +21,13 @@
import java.io.File;
-import org.junit.Test;
-
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -44,18 +42,18 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+import org.junit.Test;
public class VSizeFrameSortMergeTest extends AbstractIntegrationTest {
public static String[] INPUTS = { "data/tpch0.001/orders-part1.tbl", "data/tpch0.001/orders-part2.tbl" };
FileSplit[] ordersSplits = new FileSplit[] {
- new FileSplit(NC1_ID, new FileReference(new File(INPUTS[0]))),
- new FileSplit(NC2_ID, new FileReference(new File(INPUTS[1]))) };
+ new FileSplit(NC1_ID, new File(INPUTS[0]).getAbsolutePath(), false),
+ new FileSplit(NC2_ID, new File(INPUTS[1]).getAbsolutePath(), false) };
IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(),
@@ -97,7 +95,7 @@
File file = File.createTempFile(getClass().getName(), ".tmp");
IFileSplitProvider outputSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, file.getAbsolutePath()) });
+ new FileSplit[] { new FileSplit(NC1_ID, file.getAbsolutePath(), false) });
IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outputSplitProvider, "|");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/src/main/java/org/apache/hyracks/examples/text/client/WordCountMain.java b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/src/main/java/org/apache/hyracks/examples/text/client/WordCountMain.java
index 22a571f..5f4ac2d 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/src/main/java/org/apache/hyracks/examples/text/client/WordCountMain.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/src/main/java/org/apache/hyracks/examples/text/client/WordCountMain.java
@@ -31,7 +31,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
@@ -47,7 +47,6 @@
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.FrameFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
@@ -128,7 +127,7 @@
throw new IllegalArgumentException("File split " + s + " not well formed");
}
File file = new File(s.substring(idx + 1));
- fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(file));
+ fSplits[i] = new FileSplit(s.substring(0, idx), file.getAbsolutePath(), false);
fileSize += file.length();
}
return fSplits;
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Common.java b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Common.java
index 15d7f66..5b17b93 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Common.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Common.java
@@ -25,7 +25,7 @@
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
@@ -34,7 +34,6 @@
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
public class Common {
static RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -100,7 +99,7 @@
if (idx < 0) {
throw new IllegalArgumentException("File split " + s + " not well formed");
}
- fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(new File(s.substring(idx + 1))));
+ fSplits[i] = new FileSplit(s.substring(0, idx), new File(s.substring(idx + 1)).getAbsolutePath(), false);
}
return fSplits;
}
@@ -114,8 +113,8 @@
if (idx < 0) {
throw new IllegalArgumentException("File split " + s + " not well formed");
}
- fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(new File(s.substring(idx + 1) + "_"
- + count)));
+ fSplits[i] = new FileSplit(s.substring(0, idx), new File(s.substring(idx + 1) + "_"
+ + count).getAbsolutePath(), false);
}
return fSplits;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Groupby.java b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Groupby.java
index 2ef5835..d831e89 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Groupby.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Groupby.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.accessors.MurmurHash3BinaryHashFunctionFamily;
@@ -47,7 +48,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.FrameFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Join.java b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Join.java
index 507e1c7..9f44da6 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Join.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Join.java
@@ -36,6 +36,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
@@ -52,7 +53,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.group.HashSpillableTableFactory;
@@ -155,13 +155,13 @@
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(customerSplits);
long custFileSize = 0;
for (int i = 0; i < customerSplits.length; i++) {
- custFileSize += customerSplits[i].getLocalFile().getFile().length();
+ custFileSize += customerSplits[i].getFile(null).length();
}
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(orderSplits);
long orderFileSize = 0;
for (int i = 0; i < orderSplits.length; i++) {
- orderFileSize += orderSplits[i].getLocalFile().getFile().length();
+ orderFileSize += orderSplits[i].getFile(null).length();
}
FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitsProvider,
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Sort.java b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Sort.java
index 28e6dd9..c791f93 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Sort.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/org/apache/hyracks/examples/tpch/client/Sort.java
@@ -34,6 +34,7 @@
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
@@ -48,7 +49,6 @@
import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import org.apache.hyracks.dataflow.std.sort.AbstractSorterOperatorDescriptor;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
index 7368c61..72734d3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
@@ -21,6 +21,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
@@ -28,11 +29,12 @@
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
public class BTreeDataflowHelper extends TreeIndexDataflowHelper {
public BTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- boolean durable) {
+ boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, durable);
}
@@ -40,9 +42,11 @@
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
try {
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
return BTreeUtils.createBTree(opDesc.getStorageManager().getBufferCache(ctx), opDesc.getStorageManager()
.getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
- .getTreeIndexComparatorFactories(), BTreeLeafFrameType.REGULAR_NSM, file);
+ .getTreeIndexComparatorFactories(), BTreeLeafFrameType.REGULAR_NSM, fileRef);
} catch (BTreeException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
index 368589c..c4e3ce7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.storage.am.btree.dataflow;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -36,7 +37,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new BTreeDataflowHelper(opDesc, ctx, partition, durable);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
index 21a2441..9721097 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -69,7 +70,7 @@
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new BTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive, minFilterFieldIndexes, maxFilterFieldIndexes);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index ac59ae7..c7af3fe 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -20,6 +20,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
@@ -41,7 +42,7 @@
public BTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, int[] lowKeyFields, int[] highKeyFields,
boolean lowKeyInclusive, boolean highKeyInclusive, int[] minFilterFieldIndexes,
- int[] maxFilterFieldIndexes) {
+ int[] maxFilterFieldIndexes) throws HyracksDataException {
super(opDesc, ctx, partition, recordDescProvider, minFilterFieldIndexes, maxFilterFieldIndexes);
this.lowKeyInclusive = lowKeyInclusive;
this.highKeyInclusive = highKeyInclusive;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
index 01b6a2e..bd64174 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -54,7 +55,7 @@
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new BTreeUpdateSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive, tupleUpdaterFactory.createTupleUpdater());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
index 38ed1b1..7b81585 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
@@ -21,6 +21,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
@@ -36,7 +37,7 @@
public BTreeUpdateSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, int[] lowKeyFields, int[] highKeyFields,
- boolean lowKeyInclusive, boolean highKeyInclusive, ITupleUpdater tupleUpdater) {
+ boolean lowKeyInclusive, boolean highKeyInclusive, ITupleUpdater tupleUpdater) throws HyracksDataException {
super(opDesc, ctx, partition, recordDescProvider, lowKeyFields, highKeyFields, lowKeyInclusive,
highKeyInclusive, null, null);
this.tupleUpdater = tupleUpdater;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexDataflowHelper.java
index b39e532..261543a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexDataflowHelper.java
@@ -18,11 +18,11 @@
*/
package org.apache.hyracks.storage.am.common.api;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.common.file.LocalResource;
public interface IIndexDataflowHelper {
+
public void create() throws HyracksDataException;
/*
@@ -39,16 +39,5 @@
public IIndex getIndexInstance();
- public FileReference getFileReference();
-
- public long getResourceID() throws HyracksDataException;
-
- public IHyracksTaskContext getTaskContext();
-
- public String getResourcePath();
-
- /**
- * @return The resource unique storage partition id
- */
- public int getResourcePartition();
+ public LocalResource getResource() throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
index 05114f0..a985b4d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
@@ -23,9 +23,11 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.file.LocalResource;
+@FunctionalInterface
public interface IModificationOperationCallbackFactory extends Serializable {
- public IModificationOperationCallback createModificationOperationCallback(String resourcePath, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
index 2a1dff4..ebaee7d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
@@ -22,9 +22,10 @@
import java.io.Serializable;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
public interface IIndexDataflowHelperFactory extends Serializable {
public IIndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc,
- final IHyracksTaskContext ctx, int partition);
+ final IHyracksTaskContext ctx, int partition) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
index 56cad89..6439279 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
@@ -49,7 +49,7 @@
public IndexBulkLoadOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, IRecordDescriptorProvider recordDescProvider) {
+ boolean checkIfEmptyIndex, IRecordDescriptorProvider recordDescProvider) throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexCreateOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexCreateOperatorNodePushable.java
index 230d8ab..57c8909 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexCreateOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexCreateOperatorNodePushable.java
@@ -29,7 +29,8 @@
public class IndexCreateOperatorNodePushable extends AbstractOperatorNodePushable {
private final IIndexDataflowHelper indexHelper;
- public IndexCreateOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) {
+ public IndexCreateOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition)
+ throws HyracksDataException {
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index 202c0a8..5dbbae4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -24,11 +24,12 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrame;
-import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.common.file.ILocalResourceFactory;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
@@ -36,30 +37,31 @@
public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
- protected final IIndexOperatorDescriptor opDesc;
protected final IHyracksTaskContext ctx;
+ protected final IIOManager ioManager;
+ protected final IIndexOperatorDescriptor opDesc;
protected final IResourceLifecycleManager<IIndex> lcManager;
protected final ILocalResourceRepository localResourceRepository;
protected final IResourceIdFactory resourceIdFactory;
- protected final FileReference file;
- protected final int partition;
protected final boolean durable;
+ protected final FileReference resourceRef;
+ protected final String resourceName;
+ protected final int partition;
protected IIndex index;
- protected final String resourcePath;
- protected final int resourcePartition;
public IndexDataflowHelper(IIndexOperatorDescriptor opDesc, final IHyracksTaskContext ctx, int partition,
- boolean durable) {
- this.opDesc = opDesc;
+ boolean durable) throws HyracksDataException {
this.ctx = ctx;
+ this.opDesc = opDesc;
+ this.ioManager = ctx.getIOManager();
this.lcManager = opDesc.getLifecycleManagerProvider().getLifecycleManager(ctx);
this.localResourceRepository = opDesc.getStorageManager().getLocalResourceRepository(ctx);
this.resourceIdFactory = opDesc.getStorageManager().getResourceIdFactory(ctx);
- this.partition = partition;
- this.file = IndexFileNameUtil.getIndexAbsoluteFileRef(opDesc, partition, ctx.getIOManager());
- this.resourcePath = file.getFile().getPath();
+ FileSplit fileSplit = opDesc.getFileSplitProvider().getFileSplits()[partition];
+ this.resourceRef = ioManager.getFileRef(fileSplit.getPath(), fileSplit.isManaged());
+ this.resourceName = resourceRef.getRelativePath();
this.durable = durable;
- this.resourcePartition = opDesc.getFileSplitProvider().getFileSplits()[partition].getPartition();
+ this.partition = partition;
}
protected abstract IIndex createIndexInstance() throws HyracksDataException;
@@ -72,10 +74,10 @@
@Override
public void create() throws HyracksDataException {
synchronized (lcManager) {
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceRef.getRelativePath());
if (index != null) {
//how is this right?????????? <needs to be fixed>
- lcManager.unregister(resourcePath);
+ lcManager.unregister(resourceRef.getRelativePath());
} else {
index = createIndexInstance();
}
@@ -85,21 +87,20 @@
// Once the index has been created, a new resource ID can be generated.
long resourceID = getResourceID();
if (resourceID != -1) {
- localResourceRepository.deleteResourceByPath(resourcePath);
+ localResourceRepository.delete(resourceRef.getRelativePath());
}
index.create();
try {
resourceID = resourceIdFactory.createId();
ILocalResourceFactory localResourceFactory = opDesc.getLocalResourceFactoryProvider()
.getLocalResourceFactory();
- String resourceName = opDesc.getFileSplitProvider().getFileSplits()[partition].getLocalFile().getFile()
- .getPath();
- localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, resourceName,
- resourcePartition, LIFOMetaDataFrame.VERSION, resourcePath));
+ localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, resourceRef
+ .getRelativePath(),
+ LIFOMetaDataFrame.VERSION, partition));
} catch (IOException e) {
throw new HyracksDataException(e);
}
- lcManager.register(resourcePath, index);
+ lcManager.register(resourceRef.getRelativePath(), index);
}
}
@@ -110,62 +111,46 @@
throw new HyracksDataException("Index does not have a valid resource ID. Has it been created yet?");
}
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceRef.getRelativePath());
if (index == null) {
index = createIndexInstance();
- lcManager.register(resourcePath, index);
+ lcManager.register(resourceRef.getRelativePath(), index);
}
- lcManager.open(resourcePath);
+ lcManager.open(resourceRef.getRelativePath());
}
}
@Override
public void close() throws HyracksDataException {
synchronized (lcManager) {
- lcManager.close(resourcePath);
+ lcManager.close(resourceRef.getRelativePath());
}
}
@Override
public void destroy() throws HyracksDataException {
synchronized (lcManager) {
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceRef.getRelativePath());
if (index != null) {
- lcManager.unregister(resourcePath);
+ lcManager.unregister(resourceRef.getRelativePath());
} else {
index = createIndexInstance();
}
if (getResourceID() != -1) {
- localResourceRepository.deleteResourceByPath(resourcePath);
+ localResourceRepository.delete(resourceRef.getRelativePath());
}
index.destroy();
}
}
- @Override
- public FileReference getFileReference() {
- return file;
+ private long getResourceID() throws HyracksDataException {
+ LocalResource lr = localResourceRepository.get(resourceRef.getRelativePath());
+ return lr == null ? -1 : lr.getId();
}
@Override
- public long getResourceID() throws HyracksDataException {
- LocalResource lr = localResourceRepository.getResourceByPath(resourcePath);
- return lr == null ? -1 : lr.getResourceId();
- }
-
- @Override
- public IHyracksTaskContext getTaskContext() {
- return ctx;
- }
-
- @Override
- public String getResourcePath() {
- return resourcePath;
- }
-
- @Override
- public int getResourcePartition() {
- return resourcePartition;
+ public LocalResource getResource() throws HyracksDataException {
+ return localResourceRepository.get(resourceRef.getRelativePath());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorDescriptor.java
index 8aeac98..d5a26ea 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorDescriptor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
@@ -51,7 +52,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexDropOperatorNodePushable(this, ctx, partition);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java
index 650685b..86803ae 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java
@@ -30,7 +30,7 @@
private final IIndexDataflowHelper indexHelper;
public IndexDropOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
index 33bad73..4f01978 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
@@ -41,6 +41,7 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class IndexInsertUpdateDeleteOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
protected final IIndexOperatorDescriptor opDesc;
@@ -58,7 +59,8 @@
protected IIndex index;
public IndexInsertUpdateDeleteOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, IndexOperation op) {
+ int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, IndexOperation op)
+ throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
@@ -76,13 +78,13 @@
index = indexHelper.getIndexInstance();
try {
writer.open();
- modCallback = opDesc.getModificationOpCallbackFactory().createModificationOperationCallback(
- indexHelper.getResourcePath(), indexHelper.getResourceID(), indexHelper.getResourcePartition(),
- index, ctx, this);
+ LocalResource resource = indexHelper.getResource();
+ modCallback = opDesc.getModificationOpCallbackFactory().createModificationOperationCallback(resource, ctx,
+ this);
indexAccessor = index.createAccessor(modCallback, NoOpOperationCallback.INSTANCE);
ITupleFilterFactory tupleFilterFactory = opDesc.getTupleFilterFactory();
if (tupleFilterFactory != null) {
- tupleFilter = tupleFilterFactory.createTupleFilter(indexHelper.getTaskContext());
+ tupleFilter = tupleFilterFactory.createTupleFilter(ctx);
frameTuple = new FrameTupleReference();
}
} catch (Exception e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index 44778a2..c089854 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -73,7 +73,8 @@
protected PermutingFrameTupleReference maxFilterKey;
public IndexSearchOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IRecordDescriptorProvider recordDescProvider, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
+ IRecordDescriptorProvider recordDescProvider, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes)
+ throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
@@ -133,7 +134,7 @@
dos = tb.getDataOutput();
appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
ISearchOperationCallback searchCallback = opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(indexHelper.getResourceID(), ctx, null);
+ .createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, searchCallback);
cursor = createCursor();
if (retainInput) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
index 0823a74..30c7197 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -61,7 +62,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, fillFactor, verifyInput,
numElementsHint, checkIfEmptyIndex, recordDescProvider);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
index e3d00d5..e491d1d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -49,7 +50,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexCreateOperatorNodePushable(this, ctx, partition);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
index 6d3d7a1..07cab2e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
@@ -28,7 +28,7 @@
public abstract class TreeIndexDataflowHelper extends IndexDataflowHelper {
public TreeIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- boolean durable) {
+ boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, durable);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
index d3f1723..8589290 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -47,7 +48,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new TreeIndexDiskOrderScanOperatorNodePushable(this, ctx, partition);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
index 3ac6065..fc47548 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
@@ -34,6 +34,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class TreeIndexDiskOrderScanOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
private final AbstractTreeIndexOperatorDescriptor opDesc;
@@ -42,7 +43,7 @@
private ITreeIndex treeIndex;
public TreeIndexDiskOrderScanOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition) {
+ IHyracksTaskContext ctx, int partition) throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
@@ -56,8 +57,9 @@
try {
ITreeIndexFrame cursorFrame = treeIndex.getLeafFrameFactory().createFrame();
ITreeIndexCursor cursor = treeIndexHelper.createDiskOrderScanCursor(cursorFrame);
+ LocalResource resource = treeIndexHelper.getResource();
ISearchOperationCallback searchCallback = opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(treeIndexHelper.getResourceID(), ctx, null);
+ .createSearchOperationCallback(resource.getId(), ctx, null);
ITreeIndexAccessor indexAccessor = (ITreeIndexAccessor) treeIndex
.createAccessor(NoOpOperationCallback.INSTANCE, searchCallback);
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
index 128be56..6b55b5e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -58,7 +59,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexInsertUpdateDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation,
recordDescProvider, op);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
index 1e9b75e0..06371d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
@@ -52,7 +53,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new TreeIndexStatsOperatorNodePushable(this, ctx, partition);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index f70df29..5121f89 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -24,6 +24,8 @@
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
@@ -33,6 +35,7 @@
import org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
private final AbstractTreeIndexOperatorDescriptor opDesc;
@@ -42,7 +45,7 @@
private TreeIndexStatsGatherer statsGatherer;
public TreeIndexStatsOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
@@ -67,7 +70,10 @@
writer.open();
IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
- int indexFileId = fileMapProvider.lookupFileId(treeIndexHelper.getFileReference());
+ LocalResource resource = treeIndexHelper.getResource();
+ IIOManager ioManager = ctx.getIOManager();
+ FileReference fileRef = ioManager.getFileRef(resource.getPath(), true);
+ int indexFileId = fileMapProvider.lookupFileId(fileRef);
statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getMetaManager(), indexFileId,
treeIndex.getRootPageId());
TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
index c2cca41..90eb5a1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.common.file.LocalResource;
/**
* Dummy NoOp callback factory used primarily for testing. Always returns the {@link NoOpOperationCallback} instance.
@@ -40,9 +41,8 @@
}
@Override
- public IModificationOperationCallback createModificationOperationCallback(String resourcePath, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
- throws HyracksDataException {
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException {
return NoOpOperationCallback.INSTANCE;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
index 50f3e68..496a2a2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
@@ -21,7 +21,9 @@
import java.io.File;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
@@ -34,9 +36,9 @@
return path + File.separator + IO_DEVICE_NAME_PREFIX + ioDeviceId;
}
- public static FileReference getIndexAbsoluteFileRef(IIndexOperatorDescriptor opDesc, int partition, IIOManager ioManager){
- String indexName = opDesc.getFileSplitProvider().getFileSplits()[partition].getLocalFile().getFile().getPath();
- int ioDeviceId = opDesc.getFileSplitProvider().getFileSplits()[partition].getIODeviceId();
- return ioManager.getAbsoluteFileRef(ioDeviceId, indexName);
+ public static FileReference getIndexAbsoluteFileRef(IIndexOperatorDescriptor opDesc, int partition,
+ IIOManager ioManager) throws HyracksDataException {
+ FileSplit split = opDesc.getFileSplitProvider().getFileSplits()[partition];
+ return ioManager.getFileRef(split.getPath(), split.isManaged());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index ef8f646..9d9ed23 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -18,20 +18,19 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.dataflow;
-import java.util.List;
-
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
@@ -40,26 +39,19 @@
public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version, boolean durable) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int version, boolean durable)
+ throws HyracksDataException {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, false, null, null, null, null, durable);
this.version = version;
}
- public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version, boolean durable) {
- this(opDesc, ctx, partition, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory, needKeyDupCheck, version, durable);
- }
-
@Override
public IIndex getIndexInstance() {
synchronized (lcManager) {
if (index == null) {
try {
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceName);
} catch (HyracksDataException e) {
return null;
}
@@ -71,7 +63,10 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createExternalBTree(file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ return LSMBTreeUtils.createExternalBTree(ctx.getIOManager(), fileRef, opDesc.getStorageManager().getBufferCache(
+ ctx), opDesc
.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
index a4a0792..6334fb3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
@@ -21,6 +21,7 @@
import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -46,10 +47,10 @@
@Override
public IIndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new ExternalBTreeDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, false, version, durable);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, version, durable);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index e7a1161..95fdbf1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -20,9 +20,11 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -38,7 +40,7 @@
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- int[] buddyBtreeFields, int version, boolean durable) {
+ int[] buddyBtreeFields, int version, boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, null,
null, null, durable);
this.buddyBtreeFields = buddyBtreeFields;
@@ -48,7 +50,8 @@
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version, boolean durable) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version, boolean durable)
+ throws HyracksDataException {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, null, null, null, durable);
this.buddyBtreeFields = buddyBtreeFields;
@@ -60,7 +63,7 @@
synchronized (lcManager) {
if (index == null) {
try {
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceName);
} catch (HyracksDataException e) {
return null;
}
@@ -72,11 +75,14 @@
@Override
protected IIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createExternalBTreeWithBuddy(file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
- .getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
- .getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory
- .getOperationTracker(ctx), ioScheduler, ioOpCallbackFactory.createIOOperationCallback(),
- buddyBtreeFields, version, durable);
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ return LSMBTreeUtils.createExternalBTreeWithBuddy(ctx.getIOManager(), fileRef, opDesc.getStorageManager()
+ .getBufferCache(ctx),
+ opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
+ treeOpDesc.getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy,
+ opTrackerFactory.getOperationTracker(ctx), ioScheduler,
+ ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, version, durable);
}
public int getTargetVersion() {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
index acf502f..319b5c1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
@@ -21,6 +21,7 @@
import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -47,7 +48,7 @@
@Override
public IIndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new ExternalBTreeWithBuddyDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, buddyBtreeFields, version, durable);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 6b2f5b1..25468a3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -25,9 +25,11 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -46,7 +48,7 @@
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
- int[] filterFields, boolean durable) {
+ int[] filterFields, boolean durable) throws HyracksDataException {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
filterCmpFactories, btreeFields, filterFields, durable);
@@ -57,7 +59,7 @@
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields, boolean durable) {
+ int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.needKeyDupCheck = needKeyDupCheck;
@@ -67,7 +69,10 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx),
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ return LSMBTreeUtils.createLSMTree(ctx.getIOManager(), virtualBufferCaches, fileRef, opDesc.getStorageManager()
+ .getBufferCache(ctx),
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index 63b784a..4093938 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -54,7 +55,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new LSMBTreeDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 1175130..ca269de 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
@@ -89,7 +90,8 @@
private final ITreeIndexFrameFactory interiorFrameFactory;
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
- public ExternalBTree(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ public ExternalBTree(IIOManager ioManager, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
@@ -97,12 +99,13 @@
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree> transactionBTreeFactory, int version,
boolean durable) {
- super(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
+ super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager,
+ diskBTreeFactory,
bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount,
cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable);
this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory,
null);
- this.secondDiskComponents = new LinkedList<ILSMComponent>();
+ this.secondDiskComponents = new LinkedList<>();
this.interiorFrameFactory = interiorFrameFactory;
this.version = version;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index fd60b6d..6181299 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
@@ -94,7 +95,7 @@
private final List<ILSMComponent> secondDiskComponents;
private int version = -1;
- public ExternalBTreeWithBuddy(ITreeIndexFrameFactory btreeInteriorFrameFactory,
+ public ExternalBTreeWithBuddy(IIOManager ioManager, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ITreeIndexFrameFactory buddyBtreeLeafFrameFactory,
IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory,
@@ -103,7 +104,8 @@
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
int[] buddyBTreeFields, int version, boolean durable) {
- super(diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
+ super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker,
ioScheduler, ioOpCallback, durable);
this.btreeCmpFactories = btreeCmpFactories;
this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
@@ -115,7 +117,7 @@
bloomFilterFactory);
this.bulkComponentFactory = new LSMBTreeWithBuddyDiskComponentFactory(bulkLoadBTreeFactory, buddyBtreeFactory,
bloomFilterFactory);
- this.secondDiskComponents = new LinkedList<ILSMComponent>();
+ this.secondDiskComponents = new LinkedList<>();
this.version = version;
}
@@ -876,7 +878,7 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<String>();
+ Set<String> files = new HashSet<>();
LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
files.add(component.getBuddyBTree().getFileReference().getFile().getAbsolutePath());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 6d673a9..3f05399 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -28,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
@@ -100,7 +100,8 @@
// Primary LSMBTree has a Bloomfilter, but Secondary one doesn't have.
private final boolean hasBloomFilter;
- public LSMBTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory,
+ public LSMBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
@@ -109,8 +110,8 @@
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
- boolean durable) {
- super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ boolean durable) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
@@ -122,7 +123,7 @@
new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()), interiorFrameFactory,
insertLeafFrameFactory, cmpFactories, fieldCount,
- new FileReference(new File(fileManager.getBaseDir() + "_virtual_" + i))),
+ ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_" + i, false)),
virtualBufferCache, i == 0 ? true : false,
filterFactory == null ? null : filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
@@ -137,14 +138,16 @@
}
// Without memory components
- public LSMBTree(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ public LSMBTree(IIOManager ioManager,
+ ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) {
- super(diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate,
+ super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ bloomFilterFalsePositiveRate,
mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
@@ -483,7 +486,7 @@
}
if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
@@ -565,7 +568,7 @@
}
}
if (mergedComponent.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
@@ -882,7 +885,7 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<String>();
+ Set<String> files = new HashSet<>();
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
if (hasBloomFilter) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index f62f70c..bf6ef24 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -32,6 +32,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
@@ -45,20 +46,20 @@
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
private final boolean hasBloomFilter;
- public LSMBTreeFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public LSMBTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> btreeFactory, boolean hasBloomFilter) {
- super(fileMapProvider, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.btreeFactory = btreeFactory;
this.hasBloomFilter = hasBloomFilter;
}
@Override
- public LSMComponentFileReferences getRelFlushFileReference() {
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
- hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false), null,
+ hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING, false) : null);
}
@Override
@@ -69,8 +70,8 @@
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), null,
- hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING, false), null,
+ hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING, false) : null);
}
private static FilenameFilter btreeFilter = new FilenameFilter() {
@@ -82,9 +83,9 @@
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
- List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
- ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
// create transaction filter <to hide transaction files>
FilenameFilter transactionFilter = getTransactionFileFilter(false);
@@ -93,7 +94,7 @@
// List of valid BTree files.
cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
- HashSet<String> btreeFilesSet = new HashSet<String>();
+ HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
@@ -127,14 +128,14 @@
Collections.sort(allBloomFilterFiles);
}
- List<ComparableFileName> validComparableBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
ComparableFileName lastBTree = allBTreeFiles.get(0);
validComparableBTreeFiles.add(lastBTree);
List<ComparableFileName> validComparableBloomFilterFiles = null;
ComparableFileName lastBloomFilter = null;
if (hasBloomFilter) {
- validComparableBloomFilterFiles = new ArrayList<ComparableFileName>();
+ validComparableBloomFilterFiles = new ArrayList<>();
lastBloomFilter = allBloomFilterFiles.get(0);
validComparableBloomFilterFiles.add(lastBloomFilter);
}
@@ -204,8 +205,8 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a transaction
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
- createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false), null,
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING, false));
}
@Override
@@ -244,8 +245,8 @@
throw new HyracksDataException("unrecognized file found = " + fileName);
}
}
- FileReference bTreeFileRef = new FileReference(bTreeFile);
- FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+ FileReference bTreeFileRef = ioManager.getFileRef(bTreeFile.getAbsolutePath(), false);
+ FileReference bloomFilterFileRef = ioManager.getFileRef(bloomFilterFile.getAbsolutePath(), false);
return new LSMComponentFileReferences(bTreeFileRef, null, bloomFilterFileRef);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index ed39630..e0bebf4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
@@ -59,22 +60,23 @@
}
};
- public LSMBTreeWithBuddyFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public LSMBTreeWithBuddyFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> btreeFactory,
TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory) {
- super(fileMapProvider, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.buddyBtreeFactory = buddyBtreeFactory;
this.btreeFactory = btreeFactory;
}
@Override
- public LSMComponentFileReferences getRelFlushFileReference() {
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false),
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING, false), createFlushFile(baseName
+ + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
@@ -86,17 +88,18 @@
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
// timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
- createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createMergeFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING, false),
+ createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING, false), createMergeFile(baseName
+ + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
- List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
- ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBuddyBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBuddyBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
// Create transaction file filter
FilenameFilter transactionFilefilter = getTransactionFileFilter(false);
@@ -104,7 +107,7 @@
// Gather files.
cleanupAndGetValidFilesInternal(getCompoundFilter(btreeFilter, transactionFilefilter), btreeFactory,
allBTreeFiles);
- HashSet<String> btreeFilesSet = new HashSet<String>();
+ HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
@@ -136,15 +139,15 @@
Collections.sort(allBuddyBTreeFiles);
Collections.sort(allBloomFilterFiles);
- List<ComparableFileName> validComparableBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
ComparableFileName lastBTree = allBTreeFiles.get(0);
validComparableBTreeFiles.add(lastBTree);
- List<ComparableFileName> validComparableBuddyBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBuddyBTreeFiles = new ArrayList<>();
ComparableFileName lastBuddyBTree = allBuddyBTreeFiles.get(0);
validComparableBuddyBTreeFiles.add(lastBuddyBTree);
- List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<>();
ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
validComparableBloomFilterFiles.add(lastBloomFilter);
@@ -208,9 +211,10 @@
Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
String baseName = baseDir + ts + SPLIT_STRING + ts;
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false),
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING, false), createFlushFile(baseName
+ + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
@@ -252,9 +256,9 @@
throw new HyracksDataException("unrecognized file found = " + fileName);
}
}
- FileReference bTreeFileRef = new FileReference(bTreeFile);
- FileReference buddyBTreeFileRef = new FileReference(buddyBTreeFile);
- FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+ FileReference bTreeFileRef = ioManager.getFileRef(bTreeFile.getAbsolutePath(), false);
+ FileReference buddyBTreeFileRef = ioManager.getFileRef(buddyBTreeFile.getAbsolutePath(), false);
+ FileReference bloomFilterFileRef = ioManager.getFileRef(bloomFilterFile.getAbsolutePath(), false);
return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef, bloomFilterFileRef);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 16901e9..38a3a7bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -23,7 +23,9 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
@@ -57,12 +59,14 @@
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTreeUtils {
- public static LSMBTree createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
+ public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable)
+ throws HyracksDataException {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
@@ -77,10 +81,10 @@
IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
BloomFilterFactory bloomFilterFactory = needKeyDupCheck
@@ -98,10 +102,11 @@
}
//Primary LSMBTree index has a BloomFilter.
- ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(diskFileMapProvider, file, diskBTreeFactory,
+ ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file,
+ diskBTreeFactory,
needKeyDupCheck);
- LSMBTree lsmTree = new LSMBTree(virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
+ LSMBTree lsmTree = new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider,
typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck,
@@ -109,7 +114,8 @@
return lsmTree;
}
- public static ExternalBTree createExternalBTree(FileReference file, IBufferCache diskBufferCache,
+ public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file,
+ IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
@@ -134,32 +140,36 @@
// can be used for both inserts and deletes
ITreeIndexFrameFactory dualLeafFrameFactory = new BTreeNSMLeafFrameFactory(referencingTupleWriterFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields);
// This is the component factory for transactions
- TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+ diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, dualLeafFrameFactory, cmpFactories, typeTraits.length);
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
- ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(diskFileMapProvider, file, diskBTreeFactory,
+ ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file,
+ diskBTreeFactory,
true);
// the disk only index uses an empty ArrayList for virtual buffer caches
- ExternalBTree lsmTree = new ExternalBTree(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
+ ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory,
+ deleteLeafFrameFactory,
fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, startWithVersion, durable);
return lsmTree;
}
- public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(FileReference file, IBufferCache diskBufferCache,
+ public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file,
+ IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields,
@@ -189,11 +199,11 @@
IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length];
@@ -204,15 +214,17 @@
bloomFilterKeyFields);
// buddy b-tree factory
- TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+ diskFileMapProvider,
freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory,
buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
- ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(diskFileMapProvider, file,
+ ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file,
diskBTreeFactory, diskBuddyBTreeFactory);
// the disk only index uses an empty ArrayList for virtual buffer caches
- ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(interiorFrameFactory, insertLeafFrameFactory,
+ ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory,
+ insertLeafFrameFactory,
buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, bulkLoadBTreeFactory, diskBTreeFactory,
diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider, bloomFilterFalsePositiveRate,
mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, buddyBtreeCmpFactories,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index e97913f..a464d96 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -39,7 +39,7 @@
public void deleteDirs();
- public LSMComponentFileReferences getRelFlushFileReference();
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
index 2e2bb2c..ca61075 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
@@ -22,8 +22,10 @@
import java.util.List;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
public interface IVirtualBufferCacheProvider extends Serializable {
- public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx, IFileSplitProvider fileSplitProvider);
+ public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx,
+ IFileSplitProvider fileSplitProvider) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
index f0d1c75..fa5b623 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -51,7 +52,8 @@
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable)
+ throws HyracksDataException {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields,
durable);
@@ -62,7 +64,7 @@
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- boolean durable) {
+ boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, durable);
this.virtualBufferCaches = virtualBufferCaches;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java
index 8171c58..48f59f1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java
@@ -33,7 +33,8 @@
public class LSMIndexCompactOperatorNodePushable extends AbstractOperatorNodePushable {
private final IIndexDataflowHelper indexHelper;
- public LSMIndexCompactOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) {
+ public LSMIndexCompactOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition)
+ throws HyracksDataException {
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
}
@@ -56,7 +57,7 @@
public void initialize() throws HyracksDataException {
indexHelper.open();
ILSMIndex index = (ILSMIndex) indexHelper.getIndexInstance();
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
+ ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
try {
accessor.scheduleFullMerge(NoOpIOOperationCallback.INSTANCE);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java
index de1719c..b51d132 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java
@@ -43,7 +43,8 @@
}
public LSMIndexInsertUpdateDeleteOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, IndexOperation op) {
+ int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, IndexOperation op)
+ throws HyracksDataException {
super(opDesc, ctx, partition, fieldPermutation, recordDescProvider, op);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java
index 6835d33..e97c904 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -52,7 +53,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new LSMIndexCompactOperatorNodePushable(this, ctx, partition);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
index 8988797..090ca68 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
@@ -19,9 +19,10 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager {
- public AbortRecoverLSMIndexFileManager(FileReference file) {
- super(null, file, null);
+ public AbortRecoverLSMIndexFileManager(IIOManager ioManager, FileReference file) {
+ super(ioManager, null, file, null);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index cb3a2db..981a2c3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -29,6 +29,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
@@ -51,7 +52,7 @@
public abstract class AbstractLSMIndex implements ILSMIndexInternal {
protected final ILSMHarness lsmHarness;
-
+ protected final IIOManager ioManager;
protected final ILSMIOOperationScheduler ioScheduler;
protected final ILSMIOOperationCallback ioOpCallback;
@@ -71,16 +72,17 @@
protected final LSMComponentFilterManager filterManager;
protected final int[] filterFields;
protected final boolean durable;
-
protected boolean isActivated;
protected final AtomicBoolean[] flushRequests;
protected boolean memoryComponentsAllocated = false;
- public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache diskBufferCache,
+ public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ IBufferCache diskBufferCache,
ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
LSMComponentFilterManager filterManager, int[] filterFields, boolean durable) {
+ this.ioManager = ioManager;
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
this.diskFileMapProvider = diskFileMapProvider;
@@ -92,12 +94,12 @@
this.filterFrameFactory = filterFrameFactory;
this.filterManager = filterManager;
this.filterFields = filterFields;
- this.inactiveDiskComponents = new LinkedList<ILSMComponent>();
+ this.inactiveDiskComponents = new LinkedList<>();
this.durable = durable;
lsmHarness = new LSMHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled());
isActivated = false;
- diskComponents = new ArrayList<ILSMComponent>();
- memoryComponents = new ArrayList<ILSMComponent>();
+ diskComponents = new ArrayList<>();
+ memoryComponents = new ArrayList<>();
currentMutableComponentId = new AtomicInteger();
flushRequests = new AtomicBoolean[virtualBufferCaches.size()];
for (int i = 0; i < virtualBufferCaches.size(); i++) {
@@ -106,10 +108,11 @@
}
// The constructor used by external indexes
- public AbstractLSMIndex(IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
+ public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
boolean durable) {
+ this.ioManager = ioManager;
this.diskBufferCache = diskBufferCache;
this.diskFileMapProvider = diskFileMapProvider;
this.fileManager = fileManager;
@@ -119,8 +122,8 @@
this.durable = durable;
lsmHarness = new ExternalIndexHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled());
isActivated = false;
- diskComponents = new LinkedList<ILSMComponent>();
- this.inactiveDiskComponents = new LinkedList<ILSMComponent>();
+ diskComponents = new LinkedList<>();
+ this.inactiveDiskComponents = new LinkedList<>();
// Memory related objects are nulled
this.virtualBufferCaches = null;
memoryComponents = null;
@@ -260,7 +263,7 @@
public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMComponent> lsmComponents, boolean bulkload,
ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException {
//get set of files to be replicated for this component
- Set<String> componentFiles = new HashSet<String>();
+ Set<String> componentFiles = new HashSet<>();
//get set of files to be replicated for each component
for (ILSMComponent lsmComponent : lsmComponents) {
@@ -275,8 +278,8 @@
}
//create replication job and submit it
- LSMIndexReplicationJob job =
- new LSMIndexReplicationJob(this, ctx, componentFiles, operation, executionType, opType);
+ LSMIndexReplicationJob job = new LSMIndexReplicationJob(this, ctx, componentFiles, operation, executionType,
+ opType);
try {
diskBufferCache.getIOReplicationManager().submitJob(job);
} catch (IOException e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index ed9cad0..b9f4cad 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -35,6 +35,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.IndexException;
@@ -56,9 +57,9 @@
VALID
}
+ protected final IIOManager ioManager;
protected final IFileMapProvider fileMapProvider;
-
- // baseDir should reflect dataset name and partition name.
+ // baseDir should reflect dataset name and partition name and be absolute
protected String baseDir;
protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
protected final Comparator<String> cmp = new FileNameComparator();
@@ -67,9 +68,10 @@
private String prevTimestamp = null;
- public AbstractLSMIndexFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public AbstractLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> treeFactory) {
- this.baseDir = file.getFile().getPath();
+ this.ioManager = ioManager;
+ this.baseDir = file.getFile().getAbsolutePath();
if (!baseDir.endsWith(System.getProperty("file.separator"))) {
baseDir += System.getProperty("file.separator");
}
@@ -122,8 +124,7 @@
File dir = new File(baseDir);
String[] files = dir.list(filter);
for (String fileName : files) {
- File file = new File(dir.getPath() + File.separator + fileName);
- FileReference fileRef = new FileReference(file);
+ FileReference fileRef = ioManager.getFileRef(dir.getPath() + File.separator + fileName, false);
if (treeFactory == null) {
allFiles.add(new ComparableFileName(fileRef));
continue;
@@ -132,7 +133,7 @@
if (idxState == TreeIndexState.VALID) {
allFiles.add(new ComparableFileName(fileRef));
} else if (idxState == TreeIndexState.INVALID) {
- file.delete();
+ fileRef.delete();
}
}
}
@@ -141,7 +142,7 @@
FilenameFilter filter,
TreeIndexFactory<? extends ITreeIndex> treeFactory
) throws HyracksDataException, IndexException {
- ArrayList<ComparableFileName> tmpAllInvListsFiles = new ArrayList<ComparableFileName>();
+ ArrayList<ComparableFileName> tmpAllInvListsFiles = new ArrayList<>();
cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles);
for (ComparableFileName cmpFileName : tmpAllInvListsFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
@@ -183,19 +184,19 @@
}
};
- protected FileReference createFlushFile(String relFlushFileName) {
- return new FileReference(new File(relFlushFileName));
+ protected FileReference createFlushFile(String relFlushFileName, boolean relative) throws HyracksDataException {
+ return ioManager.getFileRef(relFlushFileName, relative);
}
- protected FileReference createMergeFile(String relMergeFileName) {
- return createFlushFile(relMergeFileName);
+ protected FileReference createMergeFile(String relMergeFileName, boolean relative) throws HyracksDataException {
+ return createFlushFile(relMergeFileName, relative);
}
@Override
- public LSMComponentFileReferences getRelFlushFileReference() {
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts), null, null);
+ return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts, false), null, null);
}
@Override
@@ -205,13 +206,13 @@
String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
// Get the range of timestamps by taking the earliest and the latest timestamps
return new LSMComponentFileReferences(createMergeFile(baseDir + firstTimestampRange[0] + SPLIT_STRING
- + lastTimestampRange[1]), null, null);
+ + lastTimestampRange[1], false), null, null);
}
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
- List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
- ArrayList<ComparableFileName> allFiles = new ArrayList<ComparableFileName>();
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allFiles = new ArrayList<>();
// Gather files and delete invalid files
// There are two types of invalid files:
@@ -232,7 +233,7 @@
// Sorts files names from earliest to latest timestamp.
Collections.sort(allFiles);
- List<ComparableFileName> validComparableFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableFiles = new ArrayList<>();
ComparableFileName last = allFiles.get(0);
validComparableFiles.add(last);
for (int i = 1; i < allFiles.size(); i++) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index c6712c1..9d17b7e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -31,10 +32,10 @@
public class BTreeFactory extends TreeIndexFactory<BTree> {
- public BTreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IMetadataManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
+ super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
cmpFactories, fieldCount);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
index 8897a2f..7ab70b66 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
@@ -19,27 +19,31 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IIndex;
+import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class IndexFactory<T extends IIndex> {
+ protected final IIOManager ioManager;
protected final IBufferCache bufferCache;
protected final IFileMapProvider fileMapProvider;
protected final IMetadataManagerFactory freePageManagerFactory;
- public IndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IMetadataManagerFactory freePageManagerFactory) {
+ this.ioManager = ioManager;
this.bufferCache = bufferCache;
this.fileMapProvider = fileMapProvider;
this.freePageManagerFactory = freePageManagerFactory;
}
- public abstract T createIndexInstance(FileReference file) throws IndexException;
+ public abstract T createIndexInstance(FileReference file) throws IndexException, HyracksDataException;
public IBufferCache getBufferCache() {
return bufferCache;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
index 7f11a8d..ff04398 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -33,10 +34,10 @@
protected final IBinaryComparatorFactory[] cmpFactories;
protected final int fieldCount;
- public TreeIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IMetadataManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(bufferCache, fileMapProvider, freePageManagerFactory);
+ super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory);
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
this.cmpFactories = cmpFactories;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java
index f1070f5..721d4aa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -64,7 +65,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, 1.0f, verifyInput,
numElementsHint, checkIfEmptyIndex, recordDescProvider);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java
index 613eaf0..0956c2e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -53,7 +54,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new LSMIndexCompactOperatorNodePushable(this, ctx, partition);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java
index f55777f..75290fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -55,7 +56,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexCreateOperatorNodePushable(this, ctx, partition);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
index 5abf4ef..8f9e843 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
@@ -24,9 +24,11 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -50,7 +52,8 @@
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) {
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
+ throws HyracksDataException {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
@@ -63,7 +66,7 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps,
- boolean durable) {
+ boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.invertedIndexFields = invertedIndexFields;
@@ -77,11 +80,14 @@
try {
IBufferCache diskBufferCache = opDesc.getStorageManager().getBufferCache(ctx);
IFileMapProvider diskFileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
- LSMInvertedIndex invIndex = InvertedIndexUtils.createLSMInvertedIndex(virtualBufferCaches,
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(invIndexOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ LSMInvertedIndex invIndex = InvertedIndexUtils.createLSMInvertedIndex(ctx.getIOManager(),
+ virtualBufferCaches,
diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(),
invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(),
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
- diskBufferCache, file.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy,
+ diskBufferCache, fileRef.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
index b30bf5b..f1cc153 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -57,7 +58,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java
index 69770e6..252a4fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -62,7 +63,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new LSMIndexInsertUpdateDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation,
recordDescProvider, op);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
index f218845..09893fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.dataflow;
-import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -38,7 +37,8 @@
public LSMInvertedIndexSearchOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, int queryFieldIndex,
- IInvertedIndexSearchModifier searchModifier, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
+ IInvertedIndexSearchModifier searchModifier, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes)
+ throws HyracksDataException {
super(opDesc, ctx, partition, recordDescProvider, minFilterFieldIndexes, maxFilterFieldIndexes);
this.searchModifier = searchModifier;
this.queryFieldIndex = queryFieldIndex;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
index a7489e7..c8ad60b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
@@ -24,9 +24,11 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -50,7 +52,8 @@
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) {
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
+ throws HyracksDataException {
this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
@@ -63,7 +66,7 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps,
- boolean durable) {
+ boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.invertedIndexFields = invertedIndexFields;
@@ -77,11 +80,14 @@
try {
IBufferCache diskBufferCache = opDesc.getStorageManager().getBufferCache(ctx);
IFileMapProvider diskFileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
- PartitionedLSMInvertedIndex invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(invIndexOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ PartitionedLSMInvertedIndex invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(ctx
+ .getIOManager(),
virtualBufferCaches, diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(),
invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(),
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
- diskBufferCache, file.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy,
+ diskBufferCache, fileRef.getFile().getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
index 4a6194b..bc1fc5b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -57,7 +58,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index bf6e9b5..4478feb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -18,7 +18,6 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -28,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
@@ -105,7 +105,7 @@
private final int[] filterFieldsForNonBulkLoadOps;
private final int[] invertedIndexFieldsForNonBulkLoadOps;
- public LSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
+ public LSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
@@ -115,8 +115,8 @@
IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps,
- boolean durable) throws IndexException {
- super(virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ boolean durable) throws IndexException, HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
@@ -139,8 +139,8 @@
BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache,
new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()),
virtualBufferCache.getFileMapProvider(), invListTypeTraits, invListCmpFactories,
- BTreeLeafFrameType.REGULAR_NSM,
- new FileReference(new File(fileManager.getBaseDir() + "_virtual_del_" + i)));
+ BTreeLeafFrameType.REGULAR_NSM, ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_del_" + i,
+ false));
LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex,
deleteKeysBTree, virtualBufferCache, i == 0 ? true : false,
filterFactory == null ? null : filterFactory.createLSMComponentFilter());
@@ -375,8 +375,8 @@
List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
int numComponents = operationalComponents.size();
boolean includeMutableComponent = false;
- ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(numComponents);
- ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<IIndexAccessor>(numComponents);
+ ArrayList<IIndexAccessor> indexAccessors = new ArrayList<>(numComponents);
+ ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<>(numComponents);
for (int i = 0; i < operationalComponents.size(); i++) {
ILSMComponent component = operationalComponents.get(i);
@@ -491,7 +491,7 @@
scanCursor.close();
}
if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
@@ -643,7 +643,7 @@
cursor.close();
}
if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
@@ -801,10 +801,10 @@
}
protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache,
- IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException {
+ IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException, HyracksDataException {
return InvertedIndexUtils.createInMemoryBTreeInvertedindex(virtualBufferCache, virtualFreePageManager,
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
- new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab_" + id)));
+ invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, ioManager
+ .getFileRef(fileManager.getBaseDir() + "_virtual_vocab_" + id, false));
}
protected LSMInvertedIndexDiskComponent createDiskInvIndexComponent(ILSMComponentFactory factory,
@@ -929,7 +929,7 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<String>();
+ Set<String> files = new HashSet<>();
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) lsmComponent;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex();
files.add(invIndex.getInvListsFile().getFile().getAbsolutePath());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index 6c699da..63772d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory;
@@ -66,19 +67,20 @@
}
};
- public LSMInvertedIndexFileManager(IFileMapProvider fileMapProvider, FileReference file, BTreeFactory btreeFactory) {
- super(fileMapProvider, file, null);
+ public LSMInvertedIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
+ BTreeFactory btreeFactory) {
+ super(ioManager, fileMapProvider, file, null);
this.btreeFactory = btreeFactory;
}
@Override
- public LSMComponentFileReferences getRelFlushFileReference() {
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX),
- createFlushFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), createFlushFile(baseName
- + SPLIT_STRING + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, false),
+ createFlushFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX, false), createFlushFile(baseName
+ + SPLIT_STRING + BLOOM_FILTER_STRING, false));
}
@Override
@@ -89,22 +91,22 @@
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX),
- createMergeFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), createMergeFile(baseName
- + SPLIT_STRING + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, false),
+ createMergeFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX, false), createMergeFile(baseName
+ + SPLIT_STRING + BLOOM_FILTER_STRING, false));
}
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
- List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
- ArrayList<ComparableFileName> allDictBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allInvListsFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allDeletedKeysBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allDictBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allInvListsFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allDeletedKeysBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
// Gather files.
cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles);
- HashSet<String> deletedKeysBTreeFilesSet = new HashSet<String>();
+ HashSet<String> deletedKeysBTreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allDeletedKeysBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
deletedKeysBTreeFilesSet.add(cmpFileName.fileName.substring(0, index));
@@ -141,15 +143,15 @@
Collections.sort(allDictBTreeFiles);
Collections.sort(allBloomFilterFiles);
- List<ComparableFileName> validComparableDictBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableDictBTreeFiles = new ArrayList<>();
ComparableFileName lastDictBTree = allDictBTreeFiles.get(0);
validComparableDictBTreeFiles.add(lastDictBTree);
- List<ComparableFileName> validComparableDeletedKeysBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableDeletedKeysBTreeFiles = new ArrayList<>();
ComparableFileName lastDeletedKeysBTree = allDeletedKeysBTreeFiles.get(0);
validComparableDeletedKeysBTreeFiles.add(lastDeletedKeysBTree);
- List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<>();
ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
validComparableBloomFilterFiles.add(lastBloomFilter);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index 7aecbd1..382d065 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -19,12 +19,12 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.io.File;
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.common.api.IVirtualMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IndexException;
@@ -46,7 +46,7 @@
public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
- public PartitionedLSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
+ public PartitionedLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
@@ -57,8 +57,9 @@
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
- throws IndexException {
- super(virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterFactory,
+ throws IndexException, HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory,
+ filterFactory,
filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider,
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields,
@@ -67,10 +68,10 @@
@Override
protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache,
- IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException {
+ IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException, HyracksDataException {
return InvertedIndexUtils.createPartitionedInMemoryBTreeInvertedindex(virtualBufferCache,
virtualFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- tokenizerFactory, new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab_" + id)));
+ tokenizerFactory, ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_vocab_" + id, false));
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
index f926bba..ab852ed 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
@@ -18,11 +18,11 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
-import java.io.File;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.IndexFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
@@ -41,11 +41,11 @@
protected final IBinaryComparatorFactory[] tokenCmpFactories;
protected final IInvertedIndexFileNameMapper fileNameMapper;
- public OnDiskInvertedIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper) {
- super(bufferCache, fileMapProvider, null);
+ super(ioManager, bufferCache, fileMapProvider, null);
this.invListBuilderFactory = invListBuilderFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
@@ -55,9 +55,9 @@
}
@Override
- public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException {
+ public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException {
String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getPath());
- FileReference invListsFile = new FileReference(new File(invListsFilePath));
+ FileReference invListsFile = ioManager.getFileRef(invListsFilePath, false);
IInvertedListBuilder invListBuilder = invListBuilderFactory.create();
return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
index b43cc39..85adb45 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
@@ -18,11 +18,11 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
-import java.io.File;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
@@ -33,18 +33,20 @@
public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFactory {
- public PartitionedOnDiskInvertedIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ public PartitionedOnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache,
+ IFileMapProvider fileMapProvider,
IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper) {
- super(bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ super(ioManager, bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
+ tokenTypeTraits,
tokenCmpFactories, fileNameMapper);
}
@Override
- public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException {
- String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getPath());
- FileReference invListsFile = new FileReference(new File(invListsFilePath));
+ public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException {
+ String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath());
+ FileReference invListsFile = ioManager.getFileRef(invListsFilePath, false);
IInvertedListBuilder invListBuilder = invListBuilderFactory.create();
return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index 87811aa..23e69c8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -19,12 +19,13 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.util;
-import java.io.File;
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
@@ -83,31 +84,35 @@
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, btreeFileRef);
}
- public static OnDiskInvertedIndex createOnDiskInvertedIndex(IBufferCache bufferCache,
+ public static OnDiskInvertedIndex createOnDiskInvertedIndex(IIOManager ioManager, IBufferCache bufferCache,
IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile) throws IndexException {
+ IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile) throws IndexException,
+ HyracksDataException {
IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- FileReference btreeFile = getBTreeFile(invListsFile);
+ FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
return new OnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile);
}
- public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IBufferCache bufferCache,
+ public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IIOManager ioManager,
+ IBufferCache bufferCache,
IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile) throws IndexException {
+ IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile) throws IndexException,
+ HyracksDataException {
IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- FileReference btreeFile = getBTreeFile(invListsFile);
+ FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile);
}
- public static FileReference getBTreeFile(FileReference invListsFile) {
- return new FileReference(new File(invListsFile.getFile().getPath() + "_btree"));
+ public static FileReference getBTreeFile(IIOManager ioManager, FileReference invListsFile)
+ throws HyracksDataException {
+ return ioManager.getFileRef(invListsFile.getFile().getPath() + "_btree", false);
}
- public static BTreeFactory createDeletedKeysBTreeFactory(IFileMapProvider diskFileMapProvider,
+ public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, IFileMapProvider diskFileMapProvider,
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
IBufferCache diskBufferCache) throws BTreeException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(invListTypeTraits);
@@ -117,23 +122,25 @@
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
LinkedListMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- BTreeFactory deletedKeysBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ BTreeFactory deletedKeysBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, leafFrameFactory, invListCmpFactories,
invListCmpFactories.length);
return deletedKeysBTreeFactory;
}
- public static LSMInvertedIndex createLSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
+ public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches,
IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- IBufferCache diskBufferCache, String onDiskDir, double bloomFilterFalsePositiveRate,
+ IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws IndexException {
+ int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws IndexException, HyracksDataException {
- BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(diskFileMapProvider, invListTypeTraits,
+ BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider,
+ invListTypeTraits,
invListCmpFactories, diskBufferCache);
int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
@@ -143,13 +150,13 @@
BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields);
- FileReference onDiskDirFileRef = new FileReference(new File(onDiskDir));
- LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(diskFileMapProvider,
+ FileReference onDiskDirFileRef = ioManager.getFileRef(absoluteOnDiskDir, false);
+ LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
onDiskDirFileRef, deletedKeysBTreeFactory);
IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(
invListTypeTraits);
- OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(diskBufferCache,
+ OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache,
diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, fileManager);
@@ -163,7 +170,8 @@
diskBufferCache.getPageSize());
filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
}
- LSMInvertedIndex invIndex = new LSMInvertedIndex(virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory,
+ LSMInvertedIndex invIndex = new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory,
+ deletedKeysBTreeFactory,
bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback,
@@ -172,18 +180,19 @@
return invIndex;
}
- public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(
+ public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager,
List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider,
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
- IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String onDiskDir,
+ IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
- throws IndexException {
+ throws IndexException, HyracksDataException {
- BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(diskFileMapProvider, invListTypeTraits,
+ BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider,
+ invListTypeTraits,
invListCmpFactories, diskBufferCache);
int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
@@ -193,13 +202,13 @@
BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields);
- FileReference onDiskDirFileRef = new FileReference(new File(onDiskDir));
- LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(diskFileMapProvider,
+ FileReference onDiskDirFileRef = ioManager.getFileRef(absoluteOnDiskDir, false);
+ LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
onDiskDirFileRef, deletedKeysBTreeFactory);
IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(
invListTypeTraits);
- PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(
+ PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager,
diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, fileManager);
@@ -213,7 +222,8 @@
diskBufferCache.getPageSize());
filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
}
- PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(virtualBufferCaches, invIndexFactory,
+ PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches,
+ invIndexFactory,
deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory, filterManager,
bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
index 7f7cafd..abd11b1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -57,7 +58,8 @@
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR)
+ throws HyracksDataException {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits,
@@ -71,7 +73,8 @@
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR)
+ throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.btreeComparatorFactories = btreeComparatorFactories;
@@ -85,7 +88,9 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx),
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ return createLSMTree(virtualBufferCaches, fileRef, opDesc.getStorageManager().getBufferCache(ctx),
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), btreeComparatorFactories,
opTrackerFactory.getOperationTracker(ctx), valueProviderFactories, rtreePolicyType, linearizeCmpFactory,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
index 9731371..c7802bf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
@@ -52,7 +52,7 @@
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version, boolean durable,
- boolean isPointMBR) {
+ boolean isPointMBR) throws HyracksDataException {
super(opDesc, ctx, partition, null, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, null, btreeFields,
null, null, null, durable, isPointMBR);
@@ -65,7 +65,7 @@
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version, boolean durable,
- boolean isPointMBR) {
+ boolean isPointMBR) throws HyracksDataException {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, linearizeCmpFactory, null, btreeFields, null, null, null, durable, isPointMBR);
@@ -77,7 +77,7 @@
synchronized (lcManager) {
if (index == null) {
try {
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceName);
} catch (HyracksDataException e) {
return null;
}
@@ -95,7 +95,8 @@
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
- return LSMRTreeUtils.createExternalRTree(file, diskBufferCache, diskFileMapProvider, typeTraits,
+ return LSMRTreeUtils.createExternalRTree(ctx.getIOManager(), file, diskBufferCache, diskFileMapProvider,
+ typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, btreeFields, version, durable,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
index 475fbac..70e0580 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -51,7 +52,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new ExternalRTreeDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index 3bb0982a..cc3b69f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -52,7 +52,7 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] btreeFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- boolean durable, boolean isPointMBR) {
+ boolean durable, boolean isPointMBR) throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, btreeComparatorFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory,
rtreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR);
@@ -67,7 +67,7 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] btreeFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- boolean durable, boolean isPointMBR) {
+ boolean durable, boolean isPointMBR) throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories,
@@ -84,7 +84,8 @@
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
- return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
+ return LSMRTreeUtils.createLSMTree(ctx.getIOManager(), virtualBufferCaches, file, diskBufferCache,
+ diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, rtreeFields, btreeFields,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index f1f9787..ed0bd2e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -70,7 +71,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new LSMRTreeDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()),
bloomFilterFalsePositiveRate, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
index 9d09ca8..e5267ef 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
@@ -50,7 +50,8 @@
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR)
+ throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, btreeComparatorFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory,
rtreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR);
@@ -65,7 +66,8 @@
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches, file, diskBufferCache,
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ctx.getIOManager(), virtualBufferCaches, file,
+ diskBufferCache,
diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, rtreeFields, filterTypeTraits,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
index 5bdcc84..725c69a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -67,7 +68,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new LSMRTreeWithAntiMatterTuplesDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()),
btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index b2a5b2b..4eb3b03 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -19,13 +19,13 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.io.File;
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IIndexCursor;
@@ -83,7 +83,7 @@
protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
protected final int[] rtreeFields;
- public AbstractLSMRTree(List<IVirtualBufferCache> virtualBufferCaches,
+ public AbstractLSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
ILSMIndexFileManager fileManager, ILSMComponentFactory componentFactory,
@@ -93,8 +93,8 @@
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFactory filterFactory,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) {
- super(virtualBufferCaches, componentFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, componentFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
int i = 0;
@@ -102,11 +102,11 @@
RTree memRTree = new RTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()), rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount,
- new FileReference(new File(fileManager.getBaseDir() + "_virtual_r_" + i)), isPointMBR);
+ ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_r_" + i, false), isPointMBR);
BTree memBTree = new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()), btreeInteriorFrameFactory,
btreeLeafFrameFactory, btreeCmpFactories, btreeCmpFactories.length,
- new FileReference(new File(fileManager.getBaseDir() + "_virtual_b_" + i)));
+ ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_b_" + i, false));
LSMRTreeMemoryComponent mutableComponent = new LSMRTreeMemoryComponent(memRTree, memBTree,
virtualBufferCache, i == 0 ? true : false,
filterFactory == null ? null : filterFactory.createLSMComponentFilter());
@@ -131,7 +131,7 @@
/*
* For External indexes with no memory components
*/
- public AbstractLSMRTree(ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+ public AbstractLSMRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileManager,
ILSMComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, int fieldCount,
@@ -140,7 +140,8 @@
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable,
boolean isPointMBR) {
- super(componentFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate,
+ super(ioManager, componentFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ bloomFilterFalsePositiveRate,
mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index fcf316b..7fbaa7f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
@@ -82,7 +83,8 @@
private int version = -1;
private final int fieldCount;
- public ExternalRTree(ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ public ExternalRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+ ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory,
TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory,
@@ -92,12 +94,13 @@
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, int version, boolean durable,
boolean isPointMBR) {
- super(rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ super(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
+ btreeLeafFrameFactory,
fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate,
diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable,
isPointMBR);
- this.secondDiskComponents = new LinkedList<ILSMComponent>();
+ this.secondDiskComponents = new LinkedList<>();
this.version = version;
this.fieldCount = fieldCount;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 60eafa4..700609a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
@@ -78,7 +79,8 @@
public class LSMRTree extends AbstractLSMRTree {
protected int[] buddyBTreeFields;
- public LSMRTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+ public LSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory,
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
@@ -89,8 +91,9 @@
ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, int[] rtreeFields, int[] buddyBTreeFields, int[] filterFields,
- boolean durable, boolean isPointMBR) {
- super(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
+ boolean durable, boolean isPointMBR) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileNameManager,
new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, filterFactory),
diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
@@ -102,7 +105,8 @@
/*
* For External indexes with no memory components
*/
- public LSMRTree(ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ public LSMRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+ ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory,
TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory,
@@ -111,7 +115,8 @@
ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, boolean durable, boolean isPointMBR) {
- super(rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ super(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
+ btreeLeafFrameFactory,
fileNameManager,
new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, null),
diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
@@ -316,7 +321,7 @@
}
if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
@@ -392,7 +397,7 @@
}
if (mergedComponent.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
@@ -633,14 +638,15 @@
@Override
public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException {
- if (!appendOnly)
+ if (!appendOnly) {
throw new UnsupportedOperationException("LSM indexes don't support in-place modification");
+ }
return createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
}
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<String>();
+ Set<String> files = new HashSet<>();
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) lsmComponent;
files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
files.add(component.getRTree().getFileReference().getFile().getAbsolutePath());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 4f50980..43d3878 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -32,6 +32,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
@@ -60,21 +61,21 @@
}
};
- public LSMRTreeFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public LSMRTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> rtreeFactory, TreeIndexFactory<? extends ITreeIndex> btreeFactory) {
- super(fileMapProvider, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.rtreeFactory = rtreeFactory;
this.btreeFactory = btreeFactory;
}
@Override
- public LSMComponentFileReferences getRelFlushFileReference() {
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + RTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + RTREE_STRING, false),
+ createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false), createFlushFile(baseName + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
@@ -86,24 +87,24 @@
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
// timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + RTREE_STRING),
- createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), createMergeFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + RTREE_STRING, false),
+ createMergeFile(baseName + SPLIT_STRING + BTREE_STRING, false), createMergeFile(baseName + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
- List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
- ArrayList<ComparableFileName> allRTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allRTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
// Create a transaction filter <- to hide transaction components->
FilenameFilter transactionFilter = getTransactionFileFilter(false);
// Gather files.
cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
- HashSet<String> btreeFilesSet = new HashSet<String>();
+ HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
@@ -133,15 +134,15 @@
Collections.sort(allBTreeFiles);
Collections.sort(allBloomFilterFiles);
- List<ComparableFileName> validComparableRTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableRTreeFiles = new ArrayList<>();
ComparableFileName lastRTree = allRTreeFiles.get(0);
validComparableRTreeFiles.add(lastRTree);
- List<ComparableFileName> validComparableBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
ComparableFileName lastBTree = allBTreeFiles.get(0);
validComparableBTreeFiles.add(lastBTree);
- List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<>();
ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
validComparableBloomFilterFiles.add(lastBloomFilter);
@@ -204,9 +205,9 @@
Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
String baseName = baseDir + ts + SPLIT_STRING + ts;
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + RTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + RTREE_STRING, false),
+ createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false), createFlushFile(baseName + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
@@ -247,9 +248,9 @@
throw new HyracksDataException("unrecognized file found = " + fileName);
}
}
- FileReference rTreeFileRef = new FileReference(rTreeFile);
- FileReference bTreeFileRef = new FileReference(bTreeFile);
- FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+ FileReference rTreeFileRef = ioManager.getFileRef(rTreeFile.getAbsolutePath(), false);
+ FileReference bTreeFileRef = ioManager.getFileRef(bTreeFile.getAbsolutePath(), false);
+ FileReference bloomFilterFileRef = ioManager.getFileRef(bloomFilterFile.getAbsolutePath(), false);
return new LSMComponentFileReferences(rTreeFileRef, bTreeFileRef, bloomFilterFileRef);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index c9c0681..2e7e0d1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
@@ -73,7 +74,7 @@
// because it should have a different tuple writer in it's leaf frames.
private final ILSMComponentFactory bulkLoaComponentFactory;
- public LSMRTreeWithAntiMatterTuples(List<IVirtualBufferCache> virtualBufferCaches,
+ public LSMRTreeWithAntiMatterTuples(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory,
@@ -83,8 +84,9 @@
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) {
- super(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
+ int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileManager,
new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(diskRTreeFactory, filterFactory),
diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
@@ -462,8 +464,9 @@
@Override
public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException {
- if (!appendOnly)
+ if (!appendOnly) {
throw new UnsupportedOperationException("LSM indexes don't support in-place modification");
+ }
return createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
index ca4e4cc..1e96fc6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
@@ -39,7 +40,8 @@
}
@Override
- public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException {
+ public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException,
+ HyracksDataException {
return new LSMRTreeDiskComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), null,
null, filterFactory == null ? null : filterFactory.createLSMComponentFilter());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java
index 6312c9a..fd1aa71 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
@@ -27,8 +28,9 @@
public class LSMRTreeWithAntiMatterTuplesFileManager extends AbstractLSMIndexFileManager {
- public LSMRTreeWithAntiMatterTuplesFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public LSMRTreeWithAntiMatterTuplesFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider,
+ FileReference file,
TreeIndexFactory<? extends ITreeIndex> rtreeFactory) {
- super(fileMapProvider, file, rtreeFactory);
+ super(ioManager, fileMapProvider, file, rtreeFactory);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
index 8e8ad95..9d93e50 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.IndexException;
@@ -34,11 +35,11 @@
private final boolean isPointMBR;
- public RTreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ public RTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IMetadataManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
boolean isPointMBR) {
- super(bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
+ super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
cmpFactories, fieldCount);
this.isPointMBR = isPointMBR;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index a1b06af..4b609bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -24,7 +24,9 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.data.std.primitive.DoublePointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
@@ -71,7 +73,8 @@
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeUtils {
- public static LSMRTree createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
+ public static LSMRTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
@@ -79,7 +82,7 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] buddyBTreeFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- boolean durable, boolean isPointMBR) throws TreeIndexException {
+ boolean durable, boolean isPointMBR) throws TreeIndexException, HyracksDataException {
int valueFieldCount = buddyBTreeFields.length;
int keyFieldCount = typeTraits.length - valueFieldCount;
@@ -113,10 +116,10 @@
LinkedListMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
typeTraits.length, isPointMBR);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories,
btreeTypeTraits.length);
@@ -140,9 +143,11 @@
diskBufferCache.getPageSize());
filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
}
- ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(diskFileMapProvider, file, diskRTreeFactory,
+ ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, file,
+ diskRTreeFactory,
diskBTreeFactory);
- LSMRTree lsmTree = new LSMRTree(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ LSMRTree lsmTree = new LSMRTree(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory,
@@ -151,7 +156,7 @@
return lsmTree;
}
- public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(
+ public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IIOManager ioManager,
List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
@@ -159,11 +164,11 @@
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizerCmpFactory, int[] rtreeFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- boolean durable, boolean isPointMBR) throws TreeIndexException {
+ boolean durable, boolean isPointMBR) throws TreeIndexException, HyracksDataException {
ITreeIndexTupleWriterFactory rtreeInteriorFrameTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits,
false);
- ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
- ITreeIndexTupleWriterFactory rtreeLeafFrameCopyTupleWriterFactory = null;
+ ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory;
+ ITreeIndexTupleWriterFactory rtreeLeafFrameCopyTupleWriterFactory;
if (isPointMBR) {
int keyFieldCount = rtreeCmpFactories.length;
int valueFieldCount = btreeCmpFactories.length - keyFieldCount;
@@ -193,11 +198,11 @@
LinkedListMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, rtreeInteriorFrameFactory, copyTupleLeafFrameFactory, rtreeCmpFactories,
typeTraits.length, isPointMBR);
- TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
typeTraits.length, isPointMBR);
@@ -228,9 +233,10 @@
diskBufferCache.getPageSize());
filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
}
- ILSMIndexFileManager fileNameManager = new LSMRTreeWithAntiMatterTuplesFileManager(diskFileMapProvider, file,
+ ILSMIndexFileManager fileNameManager = new LSMRTreeWithAntiMatterTuplesFileManager(ioManager,
+ diskFileMapProvider, file,
diskRTreeFactory);
- LSMRTreeWithAntiMatterTuples lsmTree = new LSMRTreeWithAntiMatterTuples(virtualBufferCaches,
+ LSMRTreeWithAntiMatterTuples lsmTree = new LSMRTreeWithAntiMatterTuples(ioManager, virtualBufferCaches,
rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, filterFactory, filterFrameFactory,
filterManager, diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories,
@@ -239,7 +245,8 @@
return lsmTree;
}
- public static ExternalRTree createExternalRTree(FileReference file, IBufferCache diskBufferCache,
+ public static ExternalRTree createExternalRTree(IIOManager ioManager, FileReference file,
+ IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
@@ -280,10 +287,10 @@
LinkedListMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
typeTraits.length, isPointMBR);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories,
btreeTypeTraits.length);
int[] comparatorFields = { 0 };
@@ -296,9 +303,10 @@
BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields);
- ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(diskFileMapProvider, file, diskRTreeFactory,
+ ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, file,
+ diskRTreeFactory,
diskBTreeFactory);
- ExternalRTree lsmTree = new ExternalRTree(rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ ExternalRTree lsmTree = new ExternalRTree(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length,
rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
index 78acfb8..facbf72 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
@@ -21,11 +21,13 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.am.rtree.util.RTreeUtils;
@@ -35,7 +37,8 @@
private final RTreePolicyType rtreePolicyType;
public RTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, boolean durable) {
+ IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, boolean durable)
+ throws HyracksDataException {
super(opDesc, ctx, partition, durable);
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
@@ -44,9 +47,11 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
return RTreeUtils.createRTree(treeOpDesc.getStorageManager().getBufferCache(ctx),
treeOpDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
- valueProviderFactories, treeOpDesc.getTreeIndexComparatorFactories(), rtreePolicyType, file, durable,
- false);
+ valueProviderFactories, treeOpDesc.getTreeIndexComparatorFactories(), rtreePolicyType, fileRef,
+ durable, false);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
index 6e502e9..aab4b0b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.storage.am.rtree.dataflow;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
@@ -43,7 +44,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new RTreeDataflowHelper(opDesc, ctx, partition, valueProviderFactories, rtreePolicyType, durable);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
index 42cd4a2..39ced6e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -63,7 +64,7 @@
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new RTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, keyFields,
minFilterFieldIndexes, maxFilterFieldIndexes);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
index 21319e5..1dc31b1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
@@ -21,6 +21,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
@@ -36,7 +37,7 @@
public RTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, int[] keyFields, int[] minFilterFieldIndexes,
- int[] maxFilterFieldIndexes) {
+ int[] maxFilterFieldIndexes) throws HyracksDataException {
super(opDesc, ctx, partition, recordDescProvider, minFilterFieldIndexes, maxFilterFieldIndexes);
if (keyFields != null && keyFields.length > 0) {
searchKey = new PermutingFrameTupleReference();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceFactory.java
index 1b8d617..6d5e3ab 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceFactory.java
@@ -18,7 +18,7 @@
*/
package org.apache.hyracks.storage.common.file;
+@FunctionalInterface
public interface ILocalResourceFactory {
- public LocalResource createLocalResource(long resourceId, String resourceName, int partition, int storageVersion,
- String resourcePath);
+ LocalResource createLocalResource(long id, String name, int version, int partition);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceRepository.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceRepository.java
index eef76a8..5335d9f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceRepository.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ILocalResourceRepository.java
@@ -22,11 +22,11 @@
public interface ILocalResourceRepository {
- public LocalResource getResourceByPath(String name) throws HyracksDataException;
+ LocalResource get(String name) throws HyracksDataException;
- public void insert(LocalResource resource) throws HyracksDataException;
+ void insert(LocalResource resource) throws HyracksDataException;
- public void deleteResourceByPath(String name) throws HyracksDataException;
+ void delete(String name) throws HyracksDataException;
- public long getMaxResourceID() throws HyracksDataException;
+ long maxId() throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IResourceIdFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IResourceIdFactory.java
index a7bfa10..9f67540 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IResourceIdFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IResourceIdFactory.java
@@ -20,10 +20,11 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
+@FunctionalInterface
public interface IResourceIdFactory {
/**
- * @return A unique monotonically increasing id.
+ * @return A unique id
* @throws Exception
*/
long createId() throws HyracksDataException;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/LocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/LocalResource.java
index ef69664..1cf0289 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/LocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/LocalResource.java
@@ -22,14 +22,9 @@
public class LocalResource implements Serializable {
private static final long serialVersionUID = 2L;
- private final long resourceId;
- private final String resourceName;
- private final int partition;
- private final int resourceType;
- private final Object object;
- private String resourcePath;
- private int storageVersion;
-
+ /*
+ * TODO: Types should move somewhere else
+ */
public static final int TransientResource = 0;
public static final int LSMBTreeResource = 1;
public static final int LSMRTreeResource = 2;
@@ -37,47 +32,92 @@
public static final int ExternalBTreeResource = 4;
public static final int ExternalRTreeResource = 5;
public static final int ExternalBTreeWithBuddyResource = 6;
+ /*
+ * object members
+ */
+ /**
+ * Globally unique id of the local resource
+ */
+ private final long id;
+ /**
+ * Relative path of the resource
+ */
+ private final String path;
+ /**
+ * An Integer identifying the resource type
+ */
+ private final int type;
+ /**
+ * Storage version
+ */
+ private final int version;
+ /**
+ * The serializable application dependent on the application
+ */
+ private final Serializable resource;
- public LocalResource(long resourceId, String resourceName, int partition, String resourcePath, int resourceType,
- int storageVersion, Object object) {
- this.resourceId = resourceId;
- this.resourceName = resourceName;
- this.partition = partition;
- this.resourcePath = resourcePath;
- this.resourceType = resourceType;
- this.object = object;
- this.storageVersion = storageVersion;
+ /**
+ * Construct an NC local resource
+ * @param id
+ * Globally unique id of the local resource
+ * @param path
+ * Relative path of the resource
+ * @param type
+ * An Integer identifying the resource type
+ * @param version
+ * Storage version
+ * @param resource
+ * The serializable application dependent on the application
+ */
+ public LocalResource(long id, String path, int type,
+ int version, Serializable resource) {
+ this.id = id;
+ this.path = path;
+ this.type = type;
+ this.resource = resource;
+ this.version = version;
}
- public long getResourceId() {
- return resourceId;
+ /**
+ * @return the resource id
+ */
+ public long getId() {
+ return id;
}
- public String getResourceName() {
- return resourceName;
+ /**
+ * @return the resource relative path
+ */
+ public String getPath() {
+ return path;
}
- public int getPartition() {
- return partition;
+ /**
+ * @return the resource type
+ */
+ public int getType() {
+ return type;
}
- public int getResourceType() {
- return resourceType;
+ /**
+ * @return the serializable resource
+ */
+ public Serializable getResource() {
+ return resource;
}
- public Object getResourceObject() {
- return object;
- }
-
- public String getResourcePath() {
- return resourcePath;
- }
-
+ /**
+ * @return the storage version
+ */
public int getVersion() {
- return storageVersion;
+ return version;
}
- public void setResourcePath(String resourcePath) {
- this.resourcePath = resourcePath;
+ @Override
+ public String toString() {
+ return new StringBuilder("{\"").append(LocalResource.class.getSimpleName()).append("\" : ").append("{\"id\" = ")
+ .append(id).append(", \"name\" : \"").append(path).append("\", \"type\" : ").append(type).append(
+ ", \"resource\" : ").append(resource).append(", \"version\" : ").append(version).append(" } ")
+ .toString();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ResourceIdFactoryProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ResourceIdFactoryProvider.java
index c30ba4d..c87f785 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ResourceIdFactoryProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/ResourceIdFactoryProvider.java
@@ -28,7 +28,7 @@
}
public ResourceIdFactory createResourceIdFactory() throws HyracksDataException {
- long largestResourceId = localResourceRepository.getMaxResourceID();
+ long largestResourceId = localResourceRepository.maxId();
return new ResourceIdFactory(largestResourceId);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceFactory.java
index 4ace129..e2e59d8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceFactory.java
@@ -21,9 +21,7 @@
public class TransientLocalResourceFactory implements ILocalResourceFactory {
@Override
- public LocalResource createLocalResource(long resourceId, String resourceName, int partition, int storageVersion,
- String resourcePath) {
- return new LocalResource(resourceId, resourceName, partition, resourcePath, LocalResource.TransientResource,
- storageVersion, null);
+ public LocalResource createLocalResource(long id, String name, int version, int partition) {
+ return new LocalResource(id, name, LocalResource.TransientResource, version, null);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceRepository.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceRepository.java
index 35add93..f3379c8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceRepository.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientLocalResourceRepository.java
@@ -25,37 +25,37 @@
public class TransientLocalResourceRepository implements ILocalResourceRepository {
- private Map<String, LocalResource> path2ResourceMap = new HashMap<String, LocalResource>();
- private Map<Long, LocalResource> id2ResourceMap = new HashMap<Long, LocalResource>();
+ private Map<String, LocalResource> name2ResourceMap = new HashMap<>();
+ private Map<Long, LocalResource> id2ResourceMap = new HashMap<>();
@Override
- public LocalResource getResourceByPath(String path) throws HyracksDataException {
- return path2ResourceMap.get(path);
+ public LocalResource get(String path) throws HyracksDataException {
+ return name2ResourceMap.get(path);
}
@Override
public synchronized void insert(LocalResource resource) throws HyracksDataException {
- long id = resource.getResourceId();
+ long id = resource.getId();
if (id2ResourceMap.containsKey(id)) {
throw new HyracksDataException("Duplicate resource");
}
id2ResourceMap.put(id, resource);
- path2ResourceMap.put(resource.getResourcePath(), resource);
+ name2ResourceMap.put(resource.getPath(), resource);
}
@Override
- public synchronized void deleteResourceByPath(String path) throws HyracksDataException {
- LocalResource resource = path2ResourceMap.get(path);
+ public synchronized void delete(String path) throws HyracksDataException {
+ LocalResource resource = name2ResourceMap.get(path);
if (resource == null) {
throw new HyracksDataException("Resource doesn't exist");
}
- id2ResourceMap.remove(resource.getResourceId());
- path2ResourceMap.remove(path);
+ id2ResourceMap.remove(resource.getId());
+ name2ResourceMap.remove(path);
}
@Override
- public long getMaxResourceID() throws HyracksDataException {
+ public long maxId() throws HyracksDataException {
long maxResourceId = 0;
for (Long resourceId : id2ResourceMap.keySet()) {
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index 0a9343e..6225697 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -19,6 +19,8 @@
package org.apache.hyracks.storage.am.btree;
+import static org.junit.Assert.fail;
+
import java.util.Random;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -26,6 +28,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
@@ -51,8 +54,6 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.junit.Test;
-import static org.junit.Assert.fail;
-
@SuppressWarnings("rawtypes")
public abstract class OrderedIndexExamplesTest {
protected static final Logger LOGGER = Logger.getLogger(OrderedIndexExamplesTest.class.getName());
@@ -60,7 +61,7 @@
protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields) throws TreeIndexException;
+ int[] btreeFields, int[] filterFields) throws TreeIndexException, HyracksDataException;
/**
* Fixed-Length Key,Value Example. Create a tree index with one fixed-length
@@ -101,7 +102,7 @@
}
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
int numInserts = 10000;
for (int i = 0; i < numInserts; i++) {
@@ -181,7 +182,7 @@
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
String key = "111";
@@ -262,7 +263,7 @@
}
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
int numInserts = 10000;
for (int i = 0; i < 10000; i++) {
@@ -345,7 +346,7 @@
}
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
// Max string length to be generated.
int maxLength = 10;
@@ -425,7 +426,7 @@
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
// Max string length to be generated.
int runs = 3;
@@ -532,7 +533,7 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Inserting into tree...");
}
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
@@ -639,7 +640,7 @@
LOGGER.info(ins + " tuples loaded in " + (end - start) + "ms");
}
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
// Build low key.
@@ -741,7 +742,7 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Ordered Scan:");
}
- IIndexCursor scanCursor = (IIndexCursor) indexAccessor.createSearchCursor(false);
+ IIndexCursor scanCursor = indexAccessor.createSearchCursor(false);
RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
indexAccessor.search(scanCursor, nullPred);
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
index e181710..cffb844 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
@@ -23,8 +23,6 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.junit.Test;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
@@ -39,6 +37,7 @@
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
+import org.junit.Test;
@SuppressWarnings("rawtypes")
public abstract class OrderedIndexMultiThreadTest {
@@ -58,7 +57,7 @@
protected abstract void tearDown() throws HyracksDataException;
protected abstract IIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields) throws TreeIndexException;
+ int[] bloomFilterKeyFields) throws TreeIndexException, HyracksDataException;
protected abstract IIndexTestWorkerFactory getWorkerFactory();
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
index b8f1e01..d4470fe 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
@@ -23,11 +23,10 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.junit.Test;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.DoublePointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
@@ -53,6 +52,7 @@
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
import org.apache.hyracks.storage.am.rtree.util.RTreeUtils;
+import org.junit.Test;
@SuppressWarnings("rawtypes")
public abstract class AbstractRTreeExamplesTest {
@@ -71,7 +71,8 @@
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException;
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException,
+ HyracksDataException;
/**
* Two Dimensions Example. Create an RTree index of two dimensions, where
@@ -149,7 +150,7 @@
}
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
int numInserts = 10000;
for (int i = 0; i < numInserts; i++) {
@@ -257,7 +258,7 @@
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
int p1x = rnd.nextInt();
@@ -396,7 +397,7 @@
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE,
TestOperationCallback.INSTANCE);
int p1x = rnd.nextInt();
@@ -549,7 +550,7 @@
}
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
int numInserts = 10000;
for (int i = 0; i < numInserts; i++) {
@@ -652,7 +653,7 @@
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
int runs = 3;
@@ -816,7 +817,7 @@
LOGGER.info(numInserts + " tuples loaded in " + (end - start) + "ms");
}
- IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
+ IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
// Build key.
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
index 9ab7e46..4fcd55c 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
@@ -23,8 +23,6 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.junit.Test;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
@@ -45,6 +43,7 @@
import org.apache.hyracks.storage.am.rtree.AbstractRTreeExamplesTest.RTreeType;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.am.rtree.util.RTreeUtils;
+import org.junit.Test;
@SuppressWarnings("rawtypes")
public abstract class AbstractRTreeMultiThreadTest {
@@ -74,7 +73,7 @@
protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, int[] btreeFields)
- throws TreeIndexException;
+ throws TreeIndexException, HyracksDataException;
protected abstract IIndexTestWorkerFactory getWorkerFactory();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java
index 80a69c4..4f57f19 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java
@@ -24,10 +24,6 @@
import java.util.TreeSet;
import java.util.logging.Level;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -41,11 +37,15 @@
import org.apache.hyracks.storage.am.bloomfilter.util.AbstractBloomFilterTest;
import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
@SuppressWarnings("rawtypes")
public class BloomFilterTest extends AbstractBloomFilterTest {
private final Random rnd = new Random(50);
+ @Override
@Before
public void setUp() throws HyracksDataException {
super.setUp();
@@ -81,8 +81,8 @@
// generate keys
int maxKey = 1000;
- TreeSet<Integer> uniqueKeys = new TreeSet<Integer>();
- ArrayList<Integer> keys = new ArrayList<Integer>();
+ TreeSet<Integer> uniqueKeys = new TreeSet<>();
+ ArrayList<Integer> keys = new ArrayList<>();
while (uniqueKeys.size() < numElements) {
int key = rnd.nextInt() % maxKey;
uniqueKeys.add(key);
@@ -142,10 +142,10 @@
ArrayTupleReference tuple = new ArrayTupleReference();
int maxLength = 20;
- ArrayList<String> s1 = new ArrayList<String>();
- ArrayList<String> s2 = new ArrayList<String>();
- ArrayList<String> s3 = new ArrayList<String>();
- ArrayList<String> s4 = new ArrayList<String>();
+ ArrayList<String> s1 = new ArrayList<>();
+ ArrayList<String> s2 = new ArrayList<>();
+ ArrayList<String> s3 = new ArrayList<>();
+ ArrayList<String> s4 = new ArrayList<>();
for (int i = 0; i < numElements; ++i) {
s1.add(randomString(rnd.nextInt() % maxLength, rnd));
s2.add(randomString(rnd.nextInt() % maxLength, rnd));
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java
index ffec98c..e7d493c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.bloomfilter.util;
-import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Random;
@@ -27,6 +26,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -42,6 +42,7 @@
protected final int maxOpenFiles;
protected final int hyracksFrameSize;
+ protected IIOManager ioManager;
protected IHyracksTaskContext ctx;
protected IBufferCache bufferCache;
protected IFileMapProvider fileMapProvider;
@@ -49,9 +50,6 @@
protected final Random rnd = new Random();
protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final String tmpDir = System.getProperty("java.io.tmpdir");
- protected final String sep = System.getProperty("file.separator");
- protected String fileName;
public BloomFilterTestHarness() {
this.pageSize = AccessMethodTestsConfig.BLOOM_FILTER_PAGE_SIZE;
@@ -68,12 +66,12 @@
}
public void setUp() throws HyracksDataException {
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
+ ioManager = ctx.getIOManager();
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- file = new FileReference(new File(fileName));
+ file = ioManager.getFileRef(0, simpleDateFormat.format(new Date()));
rnd.setSeed(RANDOM_SEED);
}
@@ -94,14 +92,6 @@
return fileMapProvider;
}
- public FileReference getFileReference() {
- return file;
- }
-
- public String getFileName() {
- return fileName;
- }
-
public Random getRandom() {
return rnd;
}
@@ -121,4 +111,8 @@
public int getMaxOpenFiles() {
return maxOpenFiles;
}
+
+ public FileReference getFileReference() {
+ return file;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java
index 4260aa0..dd6aca6 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.btree.util;
-import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Random;
@@ -54,7 +53,6 @@
protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final String tmpDir = System.getProperty("java.io.tmpdir");
protected final String sep = System.getProperty("file.separator");
- protected String fileName;
public BTreeTestHarness() {
this.pageSize = AccessMethodTestsConfig.BTREE_PAGE_SIZE;
@@ -71,12 +69,11 @@
}
public void setUp() throws HyracksDataException {
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- file = new FileReference(new File(fileName));
+ file = ctx.getIOManager().getFileRef(0, simpleDateFormat.format(new Date()));
rnd.setSeed(RANDOM_SEED);
}
@@ -101,10 +98,6 @@
return file;
}
- public String getFileName() {
- return fileName;
- }
-
public Random getRandom() {
return rnd;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index 686ca72..726021b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -32,6 +29,8 @@
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMBTreeBulkLoadTest extends OrderedIndexBulkLoadTest {
@@ -55,7 +54,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index c11c0f2..810ef4e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -32,6 +29,8 @@
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMBTreeDeleteTest extends OrderedIndexDeleteTest {
@@ -55,7 +54,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index b6c1002..065b0bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -21,10 +21,6 @@
import java.util.logging.Level;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
@@ -43,6 +39,9 @@
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
public class LSMBTreeExamplesTest extends OrderedIndexExamplesTest {
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@@ -50,8 +49,9 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields) throws TreeIndexException {
- return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ int[] btreeFields, int[] filterFields) throws TreeIndexException, HyracksDataException {
+ return LSMBTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index e09971a..e258e04 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -32,6 +29,8 @@
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMBTreeInsertTest extends OrderedIndexInsertTest {
@@ -55,7 +54,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
index ccbd357..f9ba603 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import org.apache.hyracks.storage.am.btree.OrderedIndexTestUtils;
import org.apache.hyracks.storage.am.common.AbstractIndexLifecycleTest;
@@ -43,7 +42,7 @@
@Override
protected boolean persistentStateExists() throws Exception {
// make sure all of the directories exist
- if (!new FileReference(harness.getFileReference().getFile()).getFile().exists()) {
+ if (!harness.getFileReference().getFile().exists()) {
return false;
}
return true;
@@ -57,7 +56,8 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ testCtx = LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, fieldSerdes.length,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index 9d3a5c5..5b09da8 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -31,6 +28,8 @@
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMBTreeMergeTest extends LSMBTreeMergeTestDriver {
@@ -54,7 +53,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index f16a37d..9132209 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -19,8 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree;
-import org.junit.Test;
-
import org.apache.hyracks.dataflow.common.util.SerdeUtils;
import org.apache.hyracks.dataflow.common.util.TupleUtils;
import org.apache.hyracks.storage.am.btree.AbstractModificationOperationCallbackTest;
@@ -31,6 +29,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpOperationTrackerProvider;
+import org.junit.Test;
public class LSMBTreeModificationOperationCallbackTest extends AbstractModificationOperationCallbackTest {
private static final int NUM_TUPLES = 11;
@@ -46,7 +45,8 @@
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ index = LSMBTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index 09c0f7c..7ba072c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -33,6 +30,8 @@
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMBTreeMultiBulkLoadTest extends OrderedIndexBulkLoadTest {
@@ -56,7 +55,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index 20dfb9c..d9b8694 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -47,12 +47,13 @@
public LSMBTreeSearchOperationCallbackTest() {
harness = new LSMBTreeTestHarness();
- deleteSet = new HashSet<Integer>();
+ deleteSet = new HashSet<>();
}
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ index = LSMBTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index fc9c450..3ddfe09 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -32,6 +29,8 @@
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMBTreeUpdateTest extends OrderedIndexUpdateTest {
@@ -55,7 +54,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index df1700a..6a6755b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -53,8 +53,9 @@
@Override
protected ITreeIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields) throws TreeIndexException {
- return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ int[] bloomFilterKeyFields) throws TreeIndexException, HyracksDataException {
+ return LSMBTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null,
@@ -68,7 +69,7 @@
@Override
protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
- ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
+ ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<>();
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
index 20825ec..ef978ab 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
@@ -21,7 +21,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
@@ -32,7 +32,7 @@
protected final float fillFactor;
public BTreeBulkLoadRunner(int numBatches, int pageSize, int numPages, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, float fillFactor) throws HyracksDataException, BTreeException {
+ IBinaryComparatorFactory[] cmpFactories, float fillFactor) throws BTreeException, HyracksException {
super(numBatches, pageSize, numPages, typeTraits, cmpFactories);
this.fillFactor = fillFactor;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
index 706fa94..0dbab03d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
@@ -35,7 +36,7 @@
protected static final int HYRACKS_FRAME_SIZE = 128;
public BTreeRunner(int numTuples, int pageSize, int numPages, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException, BTreeException {
+ IBinaryComparatorFactory[] cmpFactories) throws BTreeException, HyracksException {
super(numTuples, pageSize, numPages, typeTraits, cmpFactories);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
index bcb11d3..606ec4e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
@@ -19,14 +19,15 @@
package org.apache.hyracks.storage.am.lsm.btree.perf;
-import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
@@ -44,6 +45,7 @@
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.TransientFileMapManager;
+import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
public class InMemoryBTreeRunner extends Thread implements IExperimentRunner {
protected IBufferCache bufferCache;
@@ -58,10 +60,12 @@
protected BTree btree;
public InMemoryBTreeRunner(int numBatches, int pageSize, int numPages, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException, BTreeException {
+ IBinaryComparatorFactory[] cmpFactories) throws BTreeException, HyracksException {
this.numBatches = numBatches;
+ TestStorageManagerComponentHolder.init(pageSize, numPages, numPages);
+ IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
fileName = tmpDir + sep + simpleDateFormat.format(new Date());
- file = new FileReference(new File(fileName));
+ file = ioManager.getFileRef(fileName, false);
init(pageSize, numPages, typeTraits, cmpFactories);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index df0ce2e..52d3b58 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree.perf;
-import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -93,16 +92,15 @@
this.onDiskNumPages = onDiskNumPages;
onDiskDir = classDir + sep + simpleDateFormat.format(new Date()) + sep;
- file = new FileReference(new File(onDiskDir));
ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
TestStorageManagerComponentHolder.init(this.onDiskPageSize, this.onDiskNumPages, MAX_OPEN_FILES);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
ioManager = TestStorageManagerComponentHolder.getIOManager();
ioDeviceId = 0;
+ file = ioManager.getFileRef(onDiskDir, false);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- List<IVirtualBufferCache> virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ List<IVirtualBufferCache> virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < 2; i++) {
IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), inMemPageSize,
inMemNumPages / 2);
@@ -112,7 +110,8 @@
this.ioScheduler = AsynchronousScheduler.INSTANCE;
AsynchronousScheduler.INSTANCE.init(threadFactory);
- lsmtree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, bufferCache, fmp, typeTraits, cmpFactories,
+ lsmtree = LSMBTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, fmp, typeTraits,
+ cmpFactories,
bloomFilterKeyFields, bloomFilterFalsePositiveRate, new NoMergePolicy(), new ThreadCountingTracker(),
ioScheduler, NoOpIOOperationCallback.INSTANCE, true, null, null, null, null, true);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index c045fd7..c3f789c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.util.SerdeUtils;
import org.apache.hyracks.storage.am.btree.OrderedIndexTestContext;
import org.apache.hyracks.storage.am.common.CheckTuple;
@@ -67,7 +68,8 @@
upsertCheckTuple(checkTuple, checkTuples);
}
- public static LSMBTreeTestContext create(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
+ public static LSMBTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
int numKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback)
@@ -78,7 +80,8 @@
for (int i = 0; i < numKeyFields; ++i) {
bloomFilterKeyFields[i] = i;
}
- LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
+ LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache,
+ diskFileMapProvider,
typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallback, true, null, null, null, null, true);
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index 6fd0838..f7387dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -103,14 +103,14 @@
public void setUp() throws HyracksException {
ioManager = TestStorageManagerComponentHolder.getIOManager();
ioDeviceId = 0;
- onDiskDir = ioManager.getIODevices().get(ioDeviceId).getPath() + sep + "lsm_btree_"
+ onDiskDir = ioManager.getIODevices().get(ioDeviceId).getMount() + sep + "lsm_btree_"
+ simpleDateFormat.format(new Date()) + sep;
- file = new FileReference(new File(onDiskDir));
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
+ file = ioManager.getFileRef(onDiskDir, false);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < numMutableComponents; i++) {
IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
memNumPages / numMutableComponents);
@@ -122,8 +122,9 @@
public void tearDown() throws HyracksDataException {
diskBufferCache.close();
IODeviceHandle dev = ioManager.getIODevices().get(ioDeviceId);
- File dir = new File(dev.getPath(), onDiskDir);
+ File dir = new File(dev.getMount(), onDiskDir);
FilenameFilter filter = new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
index 7716384..87374fe 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
@@ -33,11 +34,12 @@
public class DummyLSMIndexFileManager extends AbstractLSMIndexFileManager {
- public DummyLSMIndexFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public DummyLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> treeFactory) {
- super(fileMapProvider, file, treeFactory);
+ super(ioManager, fileMapProvider, file, treeFactory);
}
+ @Override
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles)
throws HyracksDataException, IndexException {
@@ -45,7 +47,7 @@
String[] files = dir.list(filter);
for (String fileName : files) {
File file = new File(dir.getPath() + File.separator + fileName);
- FileReference fileRef = new FileReference(file);
+ FileReference fileRef = ioManager.getFileRef(file.getAbsolutePath(), false);
allFiles.add(new ComparableFileName(fileRef));
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
index fa4bd30..e1810e3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
@@ -27,7 +27,7 @@
public class DummyTreeFactory extends TreeIndexFactory<ITreeIndex> {
public DummyTreeFactory() {
- super(null, null, null, null, null, null, 0);
+ super(null, null, null, null, null, null, null, 0);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
index 12ffb13..3d3053e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
@@ -33,10 +33,6 @@
import java.util.List;
import java.util.concurrent.Executors;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.io.FileReference;
@@ -47,6 +43,9 @@
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
public class LSMIndexFileManagerTest {
private static final int DEFAULT_PAGE_SIZE = 256;
@@ -65,11 +64,11 @@
TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES);
ioManager = TestStorageManagerComponentHolder.getIOManager();
fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(null);
- baseDir = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getPath() + sep + "lsm_tree"
+ baseDir = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
+ simpleDateFormat.format(new Date()) + sep;
File f = new File(baseDir);
f.mkdirs();
- file = new FileReference(f);
+ file = ioManager.getFileRef(f.getAbsolutePath(), false);
}
@After
@@ -79,20 +78,21 @@
}
public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
- ILSMIndexFileManager fileManager = new DummyLSMIndexFileManager(fileMapProvider, file, new DummyTreeFactory());
- LinkedList<String> fileNames = new LinkedList<String>();
+ ILSMIndexFileManager fileManager = new DummyLSMIndexFileManager(ioManager, fileMapProvider, file,
+ new DummyTreeFactory());
+ LinkedList<String> fileNames = new LinkedList<>();
int numFileNames = 100;
long sleepTime = 5;
for (int i = 0; i < numFileNames; i++) {
- String flushFileName = (String) fileManager.getRelFlushFileReference().getInsertIndexFileReference()
+ String flushFileName = fileManager.getRelFlushFileReference().getInsertIndexFileReference()
.getFile().getName();
if (testFlushFileName) {
fileNames.addFirst(flushFileName);
}
Thread.sleep(sleepTime);
if (!testFlushFileName) {
- String secondFlushFileName = (String) fileManager.getRelFlushFileReference()
+ String secondFlushFileName = fileManager.getRelFlushFileReference()
.getInsertIndexFileReference().getFile().getName();
String mergeFileName = getMergeFileName(fileManager, flushFileName, secondFlushFileName);
fileNames.addFirst(mergeFileName);
@@ -100,7 +100,7 @@
}
}
- List<String> sortedFileNames = new ArrayList<String>();
+ List<String> sortedFileNames = new ArrayList<>();
sortedFileNames.addAll(fileNames);
// Make sure the comparator sorts in the correct order (i.e., the
@@ -119,11 +119,17 @@
}
public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException, IndexException {
- ILSMIndexFileManager fileManager = new DummyLSMIndexFileManager(fileMapProvider, file, new DummyTreeFactory());
+ String dirPath = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
+ + simpleDateFormat.format(new Date()) + sep;
+ File f = new File(dirPath);
+ f.mkdirs();
+ FileReference file = ioManager.getFileRef(f.getAbsolutePath(), false);
+ ILSMIndexFileManager fileManager = new DummyLSMIndexFileManager(ioManager, fileMapProvider, file,
+ new DummyTreeFactory());
fileManager.createDirs();
- List<FileReference> flushFiles = new ArrayList<FileReference>();
- List<FileReference> allFiles = new ArrayList<FileReference>();
+ List<FileReference> flushFiles = new ArrayList<>();
+ List<FileReference> allFiles = new ArrayList<>();
int numFileNames = 100;
long sleepTime = 5;
@@ -165,7 +171,7 @@
}
// Populate expected valid flush files.
- List<String> expectedValidFiles = new ArrayList<String>();
+ List<String> expectedValidFiles = new ArrayList<>();
for (int i = 30; i < 50; i++) {
expectedValidFiles.add(flushFiles.get(i).getFile().getName());
}
@@ -192,17 +198,18 @@
}
// Make sure invalid files were removed from the IODevices.
- ArrayList<String> remainingFiles = new ArrayList<String>();
- File dir = new File(baseDir);
+ ArrayList<String> remainingFiles = new ArrayList<>();
+ File dir = new File(dirPath);
FilenameFilter filter = new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
};
String[] files = dir.list(filter);
- for (String file : files) {
- File f = new File(file);
- remainingFiles.add(f.getName());
+ for (String aFilePath : files) {
+ File aFile = new File(aFilePath);
+ remainingFiles.add(aFile.getName());
}
Collections.sort(remainingFiles, fileManager.getFileNameComparator());
@@ -223,6 +230,7 @@
private void cleanDirs(IOManager ioManager) {
File dir = new File(baseDir);
FilenameFilter filter = new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
@@ -236,7 +244,7 @@
}
private IOManager createIOManager(int numDevices) throws HyracksException {
- List<IODeviceHandle> devices = new ArrayList<IODeviceHandle>();
+ List<IODeviceHandle> devices = new ArrayList<>();
for (int i = 0; i < numDevices; i++) {
String iodevPath = System.getProperty("java.io.tmpdir") + sep + "test_iodev" + i;
devices.add(new IODeviceHandle(new File(iodevPath), "wa"));
@@ -255,7 +263,7 @@
throws HyracksDataException {
File f1 = new File(firstFile);
File f2 = new File(lastFile);
- return (String) fileNameManager.getRelMergeFileReference(f1.getName(), f2.getName())
+ return fileNameManager.getRelMergeFileReference(f1.getName(), f2.getName())
.getInsertIndexFileReference().getFile().getName();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
index aa32982..a7c541a 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
@@ -20,20 +20,20 @@
import static org.junit.Assert.assertTrue;
-import java.io.File;
import java.util.Collections;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
-import org.junit.Test;
-
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
+import org.junit.Test;
public class VirtualBufferCacheTest {
private static final long SEED = 123456789L;
@@ -46,6 +46,7 @@
private final FileState[] fileStates;
private VirtualBufferCache vbc;
+ private IOManager ioManager;
public VirtualBufferCacheTest() {
fileStates = new FileState[NUM_FILES];
@@ -66,7 +67,7 @@
fileId = -1;
fileRef = null;
pinCount = 0;
- pinnedPages = new HashSet<ICachedPage>();
+ pinnedPages = new HashSet<>();
}
}
@@ -79,6 +80,8 @@
*/
@Test
public void test01() throws Exception {
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, NUM_FILES);
+ ioManager = TestStorageManagerComponentHolder.getIOManager();
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
vbc = new VirtualBufferCache(allocator, PAGE_SIZE, NUM_PAGES);
vbc.open();
@@ -110,7 +113,7 @@
for (int i = 0; i < NUM_FILES; i++) {
FileState f = fileStates[i];
String fName = String.format("f%d", i);
- f.fileRef = new FileReference(new File(fName));
+ f.fileRef = ioManager.getFileRef(fName, true);
vbc.createFile(f.fileRef);
f.fileId = vbc.getFileMapProvider().lookupFileId(f.fileRef);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
index 6ef4526..17b139e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
@@ -101,13 +101,13 @@
public void setUp() throws HyracksException {
ioManager = TestStorageManagerComponentHolder.getIOManager();
ioDeviceId = 0;
- onDiskDir = ioManager.getIODevices().get(ioDeviceId).getPath() + sep + "lsm_invertedindex_"
+ onDiskDir = ioManager.getIODevices().get(ioDeviceId).getMount() + sep + "lsm_invertedindex_"
+ simpleDateFormat.format(new Date()) + sep;
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < numMutableComponents; i++) {
IVirtualBufferCache virtualBufferCache = new MultitenantVirtualBufferCache(new VirtualBufferCache(
new HeapBufferAllocator(), memPageSize, memNumPages / numMutableComponents));
@@ -115,14 +115,15 @@
virtualBufferCache.open();
}
rnd.setSeed(RANDOM_SEED);
- invIndexFileRef = ioManager.getIODevices().get(0).createFileReference(onDiskDir + invIndexFileName);
+ invIndexFileRef = ioManager.getFileRef(onDiskDir + invIndexFileName, false);
}
public void tearDown() throws HyracksDataException {
diskBufferCache.close();
IODeviceHandle dev = ioManager.getIODevices().get(ioDeviceId);
- File dir = new File(dev.getPath(), onDiskDir);
+ File dir = new File(dev.getMount(), onDiskDir);
FilenameFilter filter = new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java
index 27063e7..42c2654 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java
@@ -19,8 +19,6 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
-import java.io.File;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.io.FileReference;
@@ -61,8 +59,9 @@
IBinaryComparatorFactory[] invListCmpFactories = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
.of(IntegerPointable.FACTORY) };
IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- FileReference btreeFile = new FileReference(new File(harness.getInvListsFileRef().getFile().getPath()
- + "_btree"));
+ FileReference btreeFile = harness.getIOManager().getFileRef(harness.getInvListsFileRef().getFile()
+ .getAbsolutePath()
+ + "_btree", false);
index = new OnDiskInvertedIndex(harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), invListBuilder,
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
harness.getInvListsFileRef(), btreeFile);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index fec5121..f236434 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -22,7 +22,6 @@
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
-import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -32,7 +31,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.util.SerdeUtils;
import org.apache.hyracks.dataflow.common.util.TupleUtils;
@@ -63,8 +62,8 @@
protected IBinaryTokenizerFactory tokenizerFactory;
protected InvertedIndexType invIndexType;
protected InvertedIndexTokenizingTupleIterator indexTupleIter;
- protected HashSet<Comparable> allTokens = new HashSet<Comparable>();
- protected List<ITupleReference> documentCorpus = new ArrayList<ITupleReference>();
+ protected HashSet<Comparable> allTokens = new HashSet<>();
+ protected List<ITupleReference> documentCorpus = new ArrayList<>();
public LSMInvertedIndexTestContext(ISerializerDeserializer[] fieldSerdes, IIndex index,
IBinaryTokenizerFactory tokenizerFactory, InvertedIndexType invIndexType,
@@ -106,6 +105,7 @@
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
int[] invertedIndexFieldsForNonBulkLoadOps) throws IndexException, HyracksDataException {
ITypeTraits[] allTypeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
+ IOManager ioManager = harness.getIOManager();
IBinaryComparatorFactory[] allCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
// Set token type traits and comparators.
@@ -131,30 +131,31 @@
invIndex = InvertedIndexUtils.createInMemoryBTreeInvertedindex(harness.getVirtualBufferCaches().get(0),
new VirtualMetaDataPageManager(harness.getVirtualBufferCaches().get(0).getNumPages()),
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
- new FileReference(new File(harness.getOnDiskDir())));
+ ioManager.getFileRef(harness.getOnDiskDir(), false));
break;
}
case PARTITIONED_INMEMORY: {
invIndex = InvertedIndexUtils.createPartitionedInMemoryBTreeInvertedindex(harness
.getVirtualBufferCaches().get(0), new VirtualMetaDataPageManager(harness.getVirtualBufferCaches()
.get(0).getNumPages()), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
- tokenCmpFactories, tokenizerFactory, new FileReference(new File(harness.getOnDiskDir())));
+ tokenCmpFactories, tokenizerFactory, ioManager.getFileRef(harness.getOnDiskDir(), false));
break;
}
case ONDISK: {
- invIndex = InvertedIndexUtils.createOnDiskInvertedIndex(harness.getDiskBufferCache(),
+ invIndex = InvertedIndexUtils.createOnDiskInvertedIndex(ioManager, harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, harness.getInvListsFileRef());
break;
}
case PARTITIONED_ONDISK: {
- invIndex = InvertedIndexUtils.createPartitionedOnDiskInvertedIndex(harness.getDiskBufferCache(),
+ invIndex = InvertedIndexUtils.createPartitionedOnDiskInvertedIndex(ioManager, harness
+ .getDiskBufferCache(),
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, harness.getInvListsFileRef());
break;
}
case LSM: {
- invIndex = InvertedIndexUtils.createLSMInvertedIndex(harness.getVirtualBufferCaches(),
+ invIndex = InvertedIndexUtils.createLSMInvertedIndex(ioManager, harness.getVirtualBufferCaches(),
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, harness.getDiskBufferCache(), harness.getOnDiskDir(),
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
@@ -164,7 +165,8 @@
break;
}
case PARTITIONED_LSM: {
- invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(harness.getVirtualBufferCaches(),
+ invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, harness
+ .getVirtualBufferCaches(),
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, harness.getDiskBufferCache(), harness.getOnDiskDir(),
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
index 9d25a5a..4d78dbe 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -34,6 +31,8 @@
import org.apache.hyracks.storage.am.rtree.AbstractRTreeBulkLoadTest;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeBulkLoadTest extends AbstractRTreeBulkLoadTest {
@@ -58,7 +57,8 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
index daeeddb..401ad63 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -34,6 +31,8 @@
import org.apache.hyracks.storage.am.rtree.AbstractRTreeDeleteTest;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeDeleteTest extends AbstractRTreeDeleteTest {
@@ -58,7 +57,8 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index ea98fdd..ac5bc64 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -44,8 +44,10 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException,
+ HyracksDataException {
+ return LSMRTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
index 7ac8ecf..803278e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -34,6 +31,8 @@
import org.apache.hyracks.storage.am.rtree.AbstractRTreeInsertTest;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeInsertTest extends AbstractRTreeInsertTest {
@@ -58,7 +57,8 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
index 29c9b11..d3650e1 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.rtree;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import org.apache.hyracks.storage.am.common.AbstractIndexLifecycleTest;
@@ -53,7 +52,7 @@
@Override
protected boolean persistentStateExists() throws Exception {
// make sure all of the directories exist
- if (!new FileReference(harness.getFileReference().getFile()).getFile().exists()) {
+ if (!harness.getFileReference().getFile().exists()) {
return false;
}
return true;
@@ -67,7 +66,8 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ testCtx = LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, RTreePolicyType.RTREE, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
index 3d8615e..bf4d503 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -33,6 +30,8 @@
import org.apache.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeMergeTest extends LSMRTreeMergeTestDriver {
@@ -57,7 +56,8 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
index a38f99e..9ea9b62 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -34,6 +31,8 @@
import org.apache.hyracks.storage.am.rtree.AbstractRTreeBulkLoadTest;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesBulkLoadTest extends AbstractRTreeBulkLoadTest {
@@ -58,7 +57,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
index a12e97f..08f828c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -34,6 +31,8 @@
import org.apache.hyracks.storage.am.rtree.AbstractRTreeDeleteTest;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesDeleteTest extends AbstractRTreeDeleteTest {
@@ -58,7 +57,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index 1d4fa4a..1146351 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -44,8 +44,9 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCaches(),
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException,
+ HyracksDataException {
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getIOManager(), harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
index 8fd6478..3c0f779 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -34,6 +31,8 @@
import org.apache.hyracks.storage.am.rtree.AbstractRTreeInsertTest;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesInsertTest extends AbstractRTreeInsertTest {
@@ -58,7 +57,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
index b6fa1cc..2154c0b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.rtree;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import org.apache.hyracks.storage.am.common.AbstractIndexLifecycleTest;
@@ -53,7 +52,7 @@
@Override
protected boolean persistentStateExists() throws Exception {
// make sure all of the directories exist
- if (!new FileReference(harness.getFileReference().getFile()).getFile().exists()) {
+ if (!harness.getFileReference().getFile().exists()) {
return false;
}
return true;
@@ -67,7 +66,8 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
+ testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness
+ .getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, RTreePolicyType.RTREE, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
index d3522f7..0306393 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
@@ -21,9 +21,6 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -33,6 +30,8 @@
import org.apache.hyracks.storage.am.lsm.rtree.util.LSMRTreeWithAntiMatterTuplesTestContext;
import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesMergeTest extends LSMRTreeMergeTestDriver {
@@ -57,7 +56,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index a41cf2f..b1b8070 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -61,8 +61,9 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException, HyracksDataException {
+ return LSMRTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
@@ -78,7 +79,7 @@
@Override
protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
- ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
+ ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<>();
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index b6df74f..2693c70 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -61,8 +61,8 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCaches(),
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException, HyracksDataException {
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getIOManager(), harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
@@ -78,7 +78,7 @@
@Override
protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
- ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
+ ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<>();
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 774d975..2fd999e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.util.SerdeUtils;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
@@ -72,7 +73,8 @@
return lsmTree.getComparatorFactories();
}
- public static LSMRTreeTestContext create(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
+ public static LSMRTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
@@ -89,7 +91,8 @@
}
IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(btreeFieldSerdes,
numBtreeFields);
- LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
+ LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache,
+ diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
index be28670..c3e86ad 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
@@ -100,14 +100,14 @@
public void setUp() throws HyracksException {
ioManager = TestStorageManagerComponentHolder.getIOManager();
ioDeviceId = 0;
- onDiskDir = ioManager.getIODevices().get(ioDeviceId).getPath() + sep + "lsm_rtree_"
+ onDiskDir = ioManager.getIODevices().get(ioDeviceId).getMount() + sep + "lsm_rtree_"
+ simpleDateFormat.format(new Date()) + sep;
- file = new FileReference(new File(onDiskDir));
+ file = ioManager.getFileRef(onDiskDir, false);
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < numMutableComponents; i++) {
IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
memNumPages / numMutableComponents);
@@ -119,8 +119,9 @@
public void tearDown() throws HyracksDataException {
diskBufferCache.close();
IODeviceHandle dev = ioManager.getIODevices().get(ioDeviceId);
- File dir = new File(dev.getPath(), onDiskDir);
+ File dir = new File(dev.getMount(), onDiskDir);
FilenameFilter filter = new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
index 29ab324..d3c82ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.util.SerdeUtils;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
@@ -72,7 +73,8 @@
return lsmTree.getComparatorFactories();
}
- public static LSMRTreeWithAntiMatterTuplesTestContext create(List<IVirtualBufferCache> virtualBufferCaches,
+ public static LSMRTreeWithAntiMatterTuplesTestContext create(IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches,
FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
int numKeyFields, RTreePolicyType rtreePolicyType, ILSMMergePolicy mergePolicy,
@@ -83,7 +85,8 @@
numKeyFields);
IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
- LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches,
+ LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager,
+ virtualBufferCaches,
file, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallback,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, null, null, null, true,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
index ca1dfce..a491b25 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
@@ -21,16 +21,16 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.am.rtree.utils.RTreeTestContext;
import org.apache.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class RTreeBulkLoadTest extends AbstractRTreeBulkLoadTest {
@@ -42,7 +42,7 @@
private final RTreeTestHarness harness = new RTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java
index 7f782aa..7169c93 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java
@@ -21,16 +21,16 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.am.rtree.utils.RTreeTestContext;
import org.apache.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class RTreeDeleteTest extends AbstractRTreeDeleteTest {
@@ -42,7 +42,7 @@
}
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java
index 6fb5b5f..9acea03 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -40,7 +41,7 @@
}
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java
index 7bd22fc..599a4dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java
@@ -21,16 +21,16 @@
import java.util.Random;
-import org.junit.After;
-import org.junit.Before;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.am.rtree.utils.RTreeTestContext;
import org.apache.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
@SuppressWarnings("rawtypes")
public class RTreeInsertTest extends AbstractRTreeInsertTest {
@@ -42,7 +42,7 @@
}
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
index e1f5744..538834b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
@@ -26,7 +26,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -69,8 +69,9 @@
this.rTreeTestUtils = new RTreeTestUtils();
}
+ @Override
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
super.setUp();
}
@@ -130,7 +131,7 @@
ITreeIndexAccessor indexAccessor = rtree.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
int numInserts = 10000;
- ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<RTreeCheckTuple>();
+ ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<>();
for (int i = 0; i < numInserts; i++) {
int p1x = rnd.nextInt();
int p1y = rnd.nextInt();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
index f6cd45a..3919a97 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.storage.am.common.IIndexTestWorkerFactory;
import org.apache.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import org.apache.hyracks.storage.am.common.TestWorkloadConf;
@@ -48,7 +49,7 @@
private final RTreeTestWorkerFactory workerFactory = new RTreeTestWorkerFactory();
@Override
- protected void setUp() throws HyracksDataException {
+ protected void setUp() throws HyracksException {
harness.setUp();
}
@@ -73,7 +74,7 @@
@Override
protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
- ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
+ ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<>();
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/AbstractRTreeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/AbstractRTreeTest.java
index 1c8a4cd..7247cd1 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/AbstractRTreeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/AbstractRTreeTest.java
@@ -21,11 +21,11 @@
import java.util.logging.Logger;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.junit.After;
import org.junit.Before;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
public abstract class AbstractRTreeTest {
protected final Logger LOGGER = Logger.getLogger(RTreeTestHarness.class.getName());
protected final RTreeTestHarness harness;
@@ -39,7 +39,7 @@
}
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
index 8c3b7c4..e84fc21 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
@@ -26,7 +26,9 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -68,11 +70,12 @@
this.hyracksFrameSize = hyracksFrameSize;
}
- public void setUp() throws HyracksDataException {
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
- file = new FileReference(new File(fileName));
- ctx = TestUtils.create(getHyracksFrameSize());
+ public void setUp() throws HyracksException {
TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
+ IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ file = ioManager.getFileRef(fileName, false);
+ ctx = TestUtils.create(getHyracksFrameSize());
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
rnd.setSeed(RANDOM_SEED);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
index baa5e76..6ca4bc4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
@@ -16,18 +16,15 @@
! specific language governing permissions and limitations
! under the License.
!-->
-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>hyracks-storage-common-test</artifactId>
<name>hyracks-storage-common-test</name>
-
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
<version>0.2.18-SNAPSHOT</version>
</parent>
-
<build>
<plugins>
<plugin>
@@ -39,11 +36,9 @@
</plugin>
</plugins>
</build>
-
<properties>
<root.dir>${basedir}/../../..</root.dir>
</properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
@@ -61,8 +56,13 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
index e85ff9e..c58d83c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.storage.common;
+import static org.junit.Assert.fail;
+
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -25,9 +27,9 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IFileHandle;
-import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IIOManager.FileReadWriteMode;
import org.apache.hyracks.api.io.IIOManager.FileSyncMode;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.common.buffercache.BufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
@@ -39,8 +41,6 @@
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.fail;
-
public class BufferCacheRegressionTest {
protected static final String tmpDir = System.getProperty("java.io.tmpdir");
protected static final String sep = System.getProperty("file.separator");
@@ -81,8 +81,9 @@
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ IOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
- FileReference firstFileRef = new FileReference(new File(fileName));
+ FileReference firstFileRef = ioManager.getFileRef(fileName, false);
bufferCache.createFile(firstFileRef);
int firstFileId = fmp.lookupFileId(firstFileRef);
bufferCache.openFile(firstFileId);
@@ -106,7 +107,7 @@
}
// Create a file with the same name.
- FileReference secondFileRef = new FileReference(new File(fileName));
+ FileReference secondFileRef = ioManager.getFileRef(fileName, false);
bufferCache.createFile(secondFileRef);
int secondFileId = fmp.lookupFileId(secondFileRef);
@@ -122,8 +123,7 @@
// ask the BufferCache to pin the page, because it would return the same
// physical memory again, and for performance reasons pages are never
// reset with 0's.
- IIOManager ioManager = ctx.getIOManager();
- FileReference testFileRef = new FileReference(new File(fileName));
+ FileReference testFileRef = ioManager.getFileRef(fileName, false);
IFileHandle testFileHandle = ioManager.open(testFileRef, FileReadWriteMode.READ_ONLY,
FileSyncMode.METADATA_SYNC_DATA_SYNC);
ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE + BufferCache.RESERVED_HEADER_BYTES);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
index 3e410ad..b96d114 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
@@ -27,22 +27,24 @@
import java.util.Map;
import java.util.Random;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Test;
-
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Test;
public class BufferCacheTest {
- protected static final List<String> openedFiles = new ArrayList<String>();
+ protected static final List<String> openedFiles = new ArrayList<>();
protected static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected static final String tmpDir = System.getProperty("java.io.tmpdir");
protected static final String sep = System.getProperty("file.separator");
@@ -62,12 +64,13 @@
}
@Test
- public void simpleOpenPinCloseTest() throws HyracksDataException {
+ public void simpleOpenPinCloseTest() throws HyracksException {
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ IOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
int num = 10;
@@ -143,16 +146,18 @@
}
@Test
- public void simpleMaxOpenFilesTest() throws HyracksDataException {
+ public void simpleMaxOpenFilesTest() throws HyracksException {
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ IOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
- List<Integer> fileIds = new ArrayList<Integer>();
+ List<Integer> fileIds = new ArrayList<>();
for (int i = 0; i < MAX_OPEN_FILES; i++) {
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
@@ -164,7 +169,7 @@
// since all files are open, next open should fail
try {
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
@@ -182,7 +187,7 @@
exceptionThrown = false;
try {
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
@@ -201,20 +206,21 @@
}
@Test
- public void contentCheckingMaxOpenFilesTest() throws HyracksDataException {
+ public void contentCheckingMaxOpenFilesTest() throws HyracksException {
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
- List<Integer> fileIds = new ArrayList<Integer>();
- Map<Integer, ArrayList<Integer>> pageContents = new HashMap<Integer, ArrayList<Integer>>();
+ List<Integer> fileIds = new ArrayList<>();
+ Map<Integer, ArrayList<Integer>> pageContents = new HashMap<>();
int num = 10;
int testPageId = 0;
// open max number of files and write some stuff into their first page
for (int i = 0; i < MAX_OPEN_FILES; i++) {
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
@@ -224,7 +230,7 @@
page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), true);
page.acquireWriteLatch();
try {
- ArrayList<Integer> values = new ArrayList<Integer>();
+ ArrayList<Integer> values = new ArrayList<>();
for (int j = 0; j < num; j++) {
int x = Math.abs(rnd.nextInt());
page.getBuffer().putInt(j * 4, x);
@@ -242,7 +248,7 @@
// since all files are open, next open should fail
try {
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
@@ -252,7 +258,7 @@
Assert.assertTrue(exceptionThrown);
// close a few random files
- ArrayList<Integer> closedFileIds = new ArrayList<Integer>();
+ ArrayList<Integer> closedFileIds = new ArrayList<>();
int filesToClose = 5;
for (int i = 0; i < filesToClose; i++) {
int ix = Math.abs(rnd.nextInt()) % fileIds.size();
@@ -264,7 +270,7 @@
// now open a few new files
for (int i = 0; i < filesToClose; i++) {
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
@@ -274,7 +280,7 @@
// since all files are open, next open should fail
try {
String fileName = getFileName();
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = ioManager.getFileRef(fileName, false);
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);