Made changes to inverted index ops according to Zach's latest changes to index lifecycle management.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1846 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
index 49f58b7..71ddf3e 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResourceManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ITupleFilterFactory;
@@ -37,6 +38,7 @@
// General.
protected final IStorageManagerInterface storageManager;
protected final IIndexLifecycleManagerProvider lifecycleManagerProvider;
+ protected final ICloseableResourceManagerProvider closeableResourceManagerProvider;
protected final boolean retainInput;
protected final IOperationCallbackProvider opCallbackProvider;
@@ -55,7 +57,8 @@
public AbstractInvertedIndexOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, int outputArity,
RecordDescriptor recDesc, IStorageManagerInterface storageManager,
IFileSplitProvider btreeFileSplitProvider, IFileSplitProvider invListsFileSplitProvider,
- IIndexLifecycleManagerProvider lifecycleManagerProvider, ITypeTraits[] tokenTypeTraits,
+ IIndexLifecycleManagerProvider lifecycleManagerProvider,
+ ICloseableResourceManagerProvider closeableResourceManagerProvider, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits,
IBinaryComparatorFactory[] invListComparatorFactories, IBinaryTokenizerFactory tokenizerFactory,
IIndexDataflowHelperFactory invertedIndexDataflowHelperFactory, boolean retainInput,
@@ -65,6 +68,7 @@
// General.
this.storageManager = storageManager;
this.lifecycleManagerProvider = lifecycleManagerProvider;
+ this.closeableResourceManagerProvider = closeableResourceManagerProvider;
this.retainInput = retainInput;
this.opCallbackProvider = opCallbackProvider;
@@ -154,4 +158,9 @@
public ITupleFilterFactory getTupleFilterFactory() {
return null;
}
+
+ @Override
+ public ICloseableResourceManagerProvider getCloseableResourceManagerProvider() {
+ return closeableResourceManagerProvider;
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
index 616b0e5..548b8c5 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResourceManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
@@ -37,14 +38,15 @@
public InvertedIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] fieldPermutation,
IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
IFileSplitProvider invListsFileSplitProvider, IIndexLifecycleManagerProvider lifecycleManagerProvider,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
- ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
- IBinaryTokenizerFactory tokenizerFactory, IIndexDataflowHelperFactory invertedIndexDataflowHelperFactory,
+ ICloseableResourceManagerProvider closeableResourceManagerProvider, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits,
+ IBinaryComparatorFactory[] invListComparatorFactories, IBinaryTokenizerFactory tokenizerFactory,
+ IIndexDataflowHelperFactory invertedIndexDataflowHelperFactory,
IOperationCallbackProvider opCallbackProvider) {
super(spec, 1, 0, null, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
- lifecycleManagerProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
- invListComparatorFactories, tokenizerFactory, invertedIndexDataflowHelperFactory, false,
- opCallbackProvider);
+ lifecycleManagerProvider, closeableResourceManagerProvider, tokenTypeTraits, tokenComparatorFactories,
+ invListsTypeTraits, invListComparatorFactories, tokenizerFactory, invertedIndexDataflowHelperFactory,
+ false, opCallbackProvider);
this.fieldPermutation = fieldPermutation;
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
index 8a6d58e..77f2326 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
@@ -23,6 +23,8 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResource;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResourceManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
@@ -33,6 +35,7 @@
private final AbstractInvertedIndexOperatorDescriptor opDesc;
private final IHyracksTaskContext ctx;
private final IIndexLifecycleManager lcManager;
+ private final ICloseableResourceManager closeableResourceManager;
private final InvertedIndexDataflowHelper invIndexDataflowHelper;
private OnDiskInvertedIndex invIndex;
private IIndexBulkLoader bulkLoader;
@@ -47,6 +50,7 @@
this.opDesc = opDesc;
this.ctx = ctx;
this.lcManager = opDesc.getLifecycleManagerProvider().getLifecycleManager(ctx);
+ this.closeableResourceManager = opDesc.getCloseableResourceManagerProvider().getCloseableResourceManager();
this.invIndexDataflowHelper = new InvertedIndexDataflowHelper(opDesc, ctx, partition);
this.recordDescProvider = recordDescProvider;
tuple.setFieldPermutation(fieldPermutation);
@@ -85,9 +89,15 @@
try {
bulkLoader.end();
} catch (IndexException e) {
- throw new HyracksDataException(e);
+ throw new HyracksDataException(e);
} finally {
- lcManager.close(invIndexDataflowHelper);
+ closeableResourceManager.addCloseableResource(ctx.getJobletContext().getJobId().getId(),
+ new ICloseableResource() {
+ @Override
+ public void close() throws HyracksDataException {
+ lcManager.close(invIndexDataflowHelper);
+ }
+ });
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexCreateOperatorDescriptor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexCreateOperatorDescriptor.java
index c81806d..75d6801 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexCreateOperatorDescriptor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexCreateOperatorDescriptor.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResourceManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
@@ -35,13 +36,14 @@
public InvertedIndexCreateOperatorDescriptor(IOperatorDescriptorRegistry spec,
IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
IFileSplitProvider invListsFileSplitProvider, IIndexLifecycleManagerProvider lifecycleManagerProvider,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
- ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
- IBinaryTokenizerFactory tokenizerFactory, IIndexDataflowHelperFactory btreeDataflowHelperFactory,
- IOperationCallbackProvider opCallbackProvider) {
+ ICloseableResourceManagerProvider closeableResourceManagerProvider, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits,
+ IBinaryComparatorFactory[] invListComparatorFactories, IBinaryTokenizerFactory tokenizerFactory,
+ IIndexDataflowHelperFactory btreeDataflowHelperFactory, IOperationCallbackProvider opCallbackProvider) {
super(spec, 0, 0, null, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
- lifecycleManagerProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
- invListComparatorFactories, tokenizerFactory, btreeDataflowHelperFactory, false, opCallbackProvider);
+ lifecycleManagerProvider, closeableResourceManagerProvider, tokenTypeTraits, tokenComparatorFactories,
+ invListsTypeTraits, invListComparatorFactories, tokenizerFactory, btreeDataflowHelperFactory, false,
+ opCallbackProvider);
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
index 5738cab..025a709 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
@@ -24,6 +24,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResourceManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
@@ -41,15 +42,16 @@
public InvertedIndexSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, int queryField,
IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
IFileSplitProvider invListsFileSplitProvider, IIndexLifecycleManagerProvider lifecycleManagerProvider,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
- ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
+ ICloseableResourceManagerProvider closeableResourceManagerProvider, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits,
+ IBinaryComparatorFactory[] invListComparatorFactories,
IIndexDataflowHelperFactory btreeDataflowHelperFactory, IBinaryTokenizerFactory queryTokenizerFactory,
IInvertedIndexSearchModifierFactory searchModifierFactory, RecordDescriptor recDesc, boolean retainInput,
IOperationCallbackProvider opCallbackProvider) {
super(spec, 1, 1, recDesc, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
- lifecycleManagerProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
- invListComparatorFactories, queryTokenizerFactory, btreeDataflowHelperFactory, retainInput,
- opCallbackProvider);
+ lifecycleManagerProvider, closeableResourceManagerProvider, tokenTypeTraits, tokenComparatorFactories,
+ invListsTypeTraits, invListComparatorFactories, queryTokenizerFactory, btreeDataflowHelperFactory,
+ retainInput, opCallbackProvider);
this.queryField = queryField;
this.searchModifierFactory = searchModifierFactory;
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
index 008c1ea..d5b8f68 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
@@ -29,6 +29,8 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResource;
+import edu.uci.ics.hyracks.storage.am.common.api.ICloseableResourceManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
@@ -42,6 +44,7 @@
private final AbstractInvertedIndexOperatorDescriptor opDesc;
private final IHyracksTaskContext ctx;
private final IIndexLifecycleManager lcManager;
+ private final ICloseableResourceManager closeableResourceManager;
private final InvertedIndexDataflowHelper invIndexDataflowHelper;
private final int queryField;
private FrameTupleAccessor accessor;
@@ -66,6 +69,7 @@
this.opDesc = opDesc;
this.ctx = ctx;
this.lcManager = opDesc.getLifecycleManagerProvider().getLifecycleManager(ctx);
+ this.closeableResourceManager = opDesc.getCloseableResourceManagerProvider().getCloseableResourceManager();
this.invIndexDataflowHelper = new InvertedIndexDataflowHelper(opDesc, ctx, partition);
this.queryField = queryField;
this.searchPred = new InvertedIndexSearchPredicate(opDesc.getTokenizerFactory().createTokenizer(),
@@ -159,7 +163,13 @@
}
writer.close();
} finally {
- lcManager.close(invIndexDataflowHelper);
+ closeableResourceManager.addCloseableResource(ctx.getJobletContext().getJobId().getId(),
+ new ICloseableResource() {
+ @Override
+ public void close() throws HyracksDataException {
+ lcManager.close(invIndexDataflowHelper);
+ }
+ });
}
}
}