Allow insert anti-matter tuples when bulk loading LSM index
Previously, when we bulk load an LSM index, we are not allowed to insert
anti-matter tuples to the disk component. However, creating secondary
index for correlated datasets requires anti-matter tuples to be inserted
as well. Thus, this patch mainly contains the following changes:
- When bulk loading LSM index, allow the user to switch between insert
mode and delete mode
- Extended the LSMDiskComponentBulkLoader with the delete method. For
LSM index with anti-matter tuples, the delete method simply sets the
TupleWriter to delete mode, and inserts the anti-matter tuple. For LSM
index with buddy btree, it simply inserts the deleted tuple into the
buddy btree.
- Since the LSMDiskComponentBulkLoader would have a delete method
anyway, added a new ILSMDiskComponentBulkLoader interface containing the
delete method.
Change-Id: I6665f56a5d2183697197298fa24824eeb827686a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1796
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index c641dc1..a11b835 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -35,13 +35,12 @@
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -448,8 +447,7 @@
// modifications
public class LSMTwoPCBTreeBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
private final ILSMDiskComponent component;
- private final IIndexBulkLoader componentBulkLoader;
- private final ITreeIndexTupleWriterFactory frameTupleWriterFactory;
+ private final ILSMDiskComponentBulkLoader componentBulkLoader;
private final boolean isTransaction;
@@ -463,9 +461,6 @@
component = createBulkLoadTarget();
}
- frameTupleWriterFactory =
- ((LSMBTreeDiskComponent) component).getBTree().getLeafFrameFactory().getTupleWriterFactory();
-
componentBulkLoader =
createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true);
}
@@ -499,9 +494,7 @@
// calling delete
@Override
public void delete(ITupleReference tuple) throws HyracksDataException {
- ((LSMBTreeRefrencingTupleWriterFactory) frameTupleWriterFactory).setMode(IndexOperation.DELETE);
- componentBulkLoader.add(tuple);
- ((LSMBTreeRefrencingTupleWriterFactory) frameTupleWriterFactory).setMode(IndexOperation.INSERT);
+ componentBulkLoader.delete(tuple);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index dfa08d6..abf3fa7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -46,6 +46,7 @@
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -275,7 +276,7 @@
}
@Override
- public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
+ public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter)
throws HyracksDataException {
BloomFilterSpecification bloomFilterSpec = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index a4c67c2..b224e87 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -48,6 +48,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -311,7 +312,7 @@
LSMBTreeDiskComponent component =
createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true);
- IIndexBulkLoader componentBulkLoader =
+ ILSMDiskComponentBulkLoader componentBulkLoader =
createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
IIndexCursor scanCursor = accessor.createSearchCursor(false);
@@ -365,7 +366,7 @@
LSMBTreeDiskComponent mergedComponent =
createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
- IIndexBulkLoader componentBulkLoader =
+ ILSMDiskComponentBulkLoader componentBulkLoader =
createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
try {
while (cursor.hasNext()) {
@@ -415,7 +416,7 @@
}
@Override
- public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
+ public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter)
throws HyracksDataException {
BloomFilterSpecification bloomFilterSpec = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
deleted file mode 100644
index 24ae68c..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.btree.tuples;
-
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-
-/**
- * This is a TupleWriterFactory that can be used to reference the last writer it created
- * and switch between actual and antimatter writer modes
- * @author alamouda
- *
- */
-public class LSMBTreeRefrencingTupleWriterFactory extends TypeAwareTupleWriterFactory{
- private static final long serialVersionUID = 1L;
- private final ITypeTraits[] typeTraits;
- private final int numKeyFields;
- private boolean isDelete;
- private LSMBTreeTupleWriter createdTupleWriter;
-
- public LSMBTreeRefrencingTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) {
- super(typeTraits);
- this.typeTraits = typeTraits;
- this.numKeyFields = numKeyFields;
- this.isDelete = isDelete;
- }
-
- @Override
- public ITreeIndexTupleWriter createTupleWriter() {
- createdTupleWriter = new LSMBTreeTupleWriter(typeTraits, numKeyFields, isDelete);
- return createdTupleWriter;
- }
-
- public void setMode(IndexOperation op){
- if(op == IndexOperation.INSERT){
- this.isDelete = false;
- if(createdTupleWriter != null){
- this.createdTupleWriter.setAntimatter(false);
- }
- } else if(op == IndexOperation.DELETE){
- this.isDelete = true;
- if(createdTupleWriter != null){
- this.createdTupleWriter.setAntimatter(true);
- }
- }
- }
-
- public LSMBTreeTupleWriter getCreatedTupleWriter() {
- return createdTupleWriter;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
index 3c60f15..502e43b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
@@ -23,8 +23,9 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
-public class LSMBTreeTupleWriter extends TypeAwareTupleWriter {
+public class LSMBTreeTupleWriter extends TypeAwareTupleWriter implements ILSMTreeTupleWriter {
private boolean isAntimatter;
private final int numKeyFields;
@@ -57,13 +58,13 @@
@Override
protected int getNullFlagsBytes(int numFields) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) numFields + 1.0) / 8.0);
+ return (int) Math.ceil((numFields + 1.0) / 8.0);
}
@Override
protected int getNullFlagsBytes(ITupleReference tuple) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0);
+ return (int) Math.ceil((tuple.getFieldCount() + 1.0) / 8.0);
}
@Override
@@ -83,8 +84,8 @@
targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
}
- // Allow using the same writer for both delete and insert tuples
- public void setAntimatter(boolean isAntimatter) {
- this.isAntimatter = isAntimatter;
+ @Override
+ public void setAntimatter(boolean isDelete) {
+ this.isAntimatter = isDelete;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
index 5ee2b81..4ea501a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
@@ -20,23 +20,24 @@
package org.apache.hyracks.storage.am.lsm.btree.tuples;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
public class LSMBTreeTupleWriterFactory extends TypeAwareTupleWriterFactory {
private static final long serialVersionUID = 1L;
private final int numKeyFields;
- private final boolean isDelete;
+ private final boolean isAntimatter;
- public LSMBTreeTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) {
+ public LSMBTreeTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isAntimatter) {
super(typeTraits);
this.numKeyFields = numKeyFields;
- this.isDelete = isDelete;
+ this.isAntimatter = isAntimatter;
}
@Override
- public ITreeIndexTupleWriter createTupleWriter() {
- return new LSMBTreeTupleWriter(typeTraits, numKeyFields, isDelete);
+ public ILSMTreeTupleWriter createTupleWriter() {
+ return new LSMBTreeTupleWriter(typeTraits, numKeyFields, isAntimatter);
}
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
index 0e5fcdb..259f25f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
@@ -39,7 +39,6 @@
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory;
-import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -74,16 +73,21 @@
new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, true);
LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory =
new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length);
+ LSMBTreeTupleWriterFactory bulkLoadTupleWriterFactory =
+ new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
+
ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
+ ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory);
TreeIndexFactory<BTree> diskBTreeFactory =
new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
- freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
BloomFilterFactory bloomFilterFactory = needKeyDupCheck
? new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields) : null;
@@ -125,11 +129,12 @@
ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
// This is the tuple writer that can do both inserts and deletes
- LSMBTreeRefrencingTupleWriterFactory referencingTupleWriterFactory =
- new LSMBTreeRefrencingTupleWriterFactory(typeTraits, cmpFactories.length, false);
+ LSMBTreeTupleWriterFactory transactionTupleWriterFactory =
+ new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
// This is the leaf frame factory for transaction components since it
// can be used for both inserts and deletes
- ITreeIndexFrameFactory dualLeafFrameFactory = new BTreeNSMLeafFrameFactory(referencingTupleWriterFactory);
+ ITreeIndexFrameFactory transactionLeafFrameFactory =
+ new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory);
TreeIndexFactory<BTree> diskBTreeFactory =
new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
@@ -143,7 +148,7 @@
// This is the component factory for transactions
TreeIndexFactory<BTree> transactionBTreeFactory =
new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
- interiorFrameFactory, dualLeafFrameFactory, cmpFactories, typeTraits.length);
+ interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length);
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
ILSMIndexFileManager fileNameManager =
new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java
new file mode 100644
index 0000000..f1d1ce1
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+
+public interface ILSMDiskComponentBulkLoader extends IIndexBulkLoader {
+
+ /**
+ * Delete the given tuple (insert an antimatter tuple or deleted-key tuple)
+ * from the disk component
+ *
+ * @param tuple
+ * @throws HyracksDataException
+ */
+ void delete(ITupleReference tuple) throws HyracksDataException;
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 5a6f391..5b3872c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -27,7 +27,6 @@
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness;
import org.apache.hyracks.storage.common.IIndex;
-import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
@@ -153,7 +152,8 @@
* @return
* @throws HyracksDataException
*/
- IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, boolean verifyInput,
- long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) throws HyracksDataException;
+ ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
+ boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java
new file mode 100644
index 0000000..1d6201d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+
+/**
+ * This interface allows to write both insert (matter) and delete (anti-matter) tuples
+ *
+ * @author luochen
+ *
+ */
+public interface ILSMTreeTupleWriter extends ITreeIndexTupleWriter {
+
+ void setAntimatter(boolean isAntimatter);
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java
index 964893a..ad47bbc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java
@@ -23,14 +23,17 @@
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex.AbstractTreeIndexBulkLoader;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.MultiComparator;
-public abstract class AbstractLSMDiskComponentBulkLoader implements IIndexBulkLoader {
+public abstract class AbstractLSMDiskComponentBulkLoader implements ILSMDiskComponentBulkLoader {
protected final ILSMDiskComponent component;
protected final IIndexBulkLoader indexBulkLoader;
@@ -87,11 +90,8 @@
if (bloomFilterBuilder != null) {
bloomFilterBuilder.add(t);
}
+ updateFilter(tuple);
- if (filterTuple != null) {
- filterTuple.reset(tuple);
- component.getLSMComponentFilter().update(filterTuple, filterCmp);
- }
} catch (Exception e) {
cleanupArtifacts();
throw e;
@@ -102,6 +102,34 @@
}
@Override
+ public void delete(ITupleReference tuple) throws HyracksDataException {
+ ILSMTreeTupleWriter tupleWriter =
+ (ILSMTreeTupleWriter) ((AbstractTreeIndexBulkLoader) indexBulkLoader).getLeafFrame().getTupleWriter();
+ tupleWriter.setAntimatter(true);
+ try {
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ indexBulkLoader.add(t);
+
+ updateFilter(tuple);
+ } catch (Exception e) {
+ cleanupArtifacts();
+ throw e;
+ } finally {
+ tupleWriter.setAntimatter(false);
+ }
+ if (isEmptyComponent) {
+ isEmptyComponent = false;
+ }
+ }
+
+ @Override
public void abort() throws HyracksDataException {
if (indexBulkLoader != null) {
indexBulkLoader.abort();
@@ -148,6 +176,13 @@
}
}
+ protected void updateFilter(ITupleReference tuple) throws HyracksDataException {
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
+ }
+
/**
* TreeIndex is used to hold the filter tuple values
*
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java
index 453d6cf..d3eafdc6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java
@@ -59,10 +59,7 @@
indexBulkLoader.add(t);
- if (filterTuple != null) {
- filterTuple.reset(tuple);
- component.getLSMComponentFilter().update(filterTuple, filterCmp);
- }
+ updateFilter(tuple);
} catch (Exception e) {
cleanupArtifacts();
throw e;
@@ -72,12 +69,23 @@
}
}
+ @Override
public void delete(ITupleReference tuple) throws HyracksDataException {
try {
- buddyBTreeBulkLoader.add(tuple);
- if (bloomFilterBuilder != null) {
- bloomFilterBuilder.add(tuple);
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
}
+
+ buddyBTreeBulkLoader.add(t);
+ if (bloomFilterBuilder != null) {
+ bloomFilterBuilder.add(t);
+ }
+
+ updateFilter(tuple);
} catch (HyracksDataException e) {
//deleting a key multiple times is OK
if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index f827b21..bd4797a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -49,6 +49,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -362,7 +363,7 @@
btreeCountingCursor.close();
}
- IIndexBulkLoader componentBulkLoader =
+ ILSMDiskComponentBulkLoader componentBulkLoader =
createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
// Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
@@ -425,7 +426,7 @@
LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(),
mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
- IIndexBulkLoader componentBulkLoader;
+ ILSMDiskComponentBulkLoader componentBulkLoader;
// In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted indexes so that
// lsmHarness.endSearch() is called once when the inverted indexes have been merged.
@@ -448,7 +449,7 @@
while (btreeCursor.hasNext()) {
btreeCursor.next();
ITupleReference tuple = btreeCursor.getTuple();
- ((LSMInvertedIndexDiskComponentBulkLoader) componentBulkLoader).delete(tuple);
+ componentBulkLoader.delete(tuple);
}
} finally {
btreeCursor.close();
@@ -489,7 +490,7 @@
}
@Override
- public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
+ public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter)
throws HyracksDataException {
BloomFilterSpecification bloomFilterSpec = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 3618737..eb838a9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -49,6 +49,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -195,7 +196,7 @@
btreeCountingCursor.close();
}
- IIndexBulkLoader componentBulkLoader =
+ ILSMDiskComponentBulkLoader componentBulkLoader =
createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
ITreeIndexCursor cursor;
@@ -240,7 +241,7 @@
while (btreeScanCursor.hasNext()) {
btreeScanCursor.next();
ITupleReference frameTuple = btreeScanCursor.getTuple();
- ((LSMRTreeDiskComponentBulkLoader) componentBulkLoader).delete(frameTuple);
+ componentBulkLoader.delete(frameTuple);
}
} finally {
btreeScanCursor.close();
@@ -272,7 +273,7 @@
LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
- IIndexBulkLoader componentBulkLoader;
+ ILSMDiskComponentBulkLoader componentBulkLoader;
// In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the r-trees so that
// lsmHarness.endSearch() is called once when the r-trees have been merged.
@@ -293,7 +294,7 @@
while (btreeCursor.hasNext()) {
btreeCursor.next();
ITupleReference tuple = btreeCursor.getTuple();
- ((LSMRTreeDiskComponentBulkLoader) componentBulkLoader).delete(tuple);
+ componentBulkLoader.delete(tuple);
}
} finally {
btreeCursor.close();
@@ -343,7 +344,7 @@
}
@Override
- public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
+ public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter)
throws HyracksDataException {
BloomFilterSpecification bloomFilterSpec = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 24c46d7..81fd981 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -39,6 +39,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -135,7 +136,8 @@
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
- IIndexBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
+ ILSMDiskComponentBulkLoader componentBulkLoader =
+ createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
// Since the LSM-RTree is used as a secondary assumption, the
// primary key will be the last comparator in the BTree comparators
@@ -221,7 +223,8 @@
// Bulk load the tuples from all on-disk RTrees into the new RTree.
LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
- IIndexBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
+ ILSMDiskComponentBulkLoader componentBulkLoader =
+ createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
try {
while (cursor.hasNext()) {
cursor.next();
@@ -254,7 +257,7 @@
}
@Override
- public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
+ public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter)
throws HyracksDataException {
if (withFilter && filterFields != null) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
index 3e586e6..83eda02 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
@@ -22,10 +22,11 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
import org.apache.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
-public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter {
- private final boolean isAntimatter;
+public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter implements ILSMTreeTupleWriter {
+ private boolean isAntimatter;
public LSMRTreeTupleWriter(ITypeTraits[] typeTraits, boolean isAntimatter) {
super(typeTraits);
@@ -54,13 +55,13 @@
@Override
protected int getNullFlagsBytes(int numFields) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) numFields + 1.0) / 8.0);
+ return (int) Math.ceil((numFields + 1.0) / 8.0);
}
@Override
protected int getNullFlagsBytes(ITupleReference tuple) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0);
+ return (int) Math.ceil((tuple.getFieldCount() + 1.0) / 8.0);
}
protected void setAntimatterBit(byte[] targetBuf, int targetOff) {
@@ -68,4 +69,9 @@
targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
}
+ @Override
+ public void setAntimatter(boolean isAntimatter) {
+ this.isAntimatter = isAntimatter;
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
index 0c21194..4d06d84 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
@@ -20,23 +20,24 @@
package org.apache.hyracks.storage.am.lsm.rtree.tuples;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
public class LSMRTreeTupleWriterFactory extends TypeAwareTupleWriterFactory {
private static final long serialVersionUID = 1L;
private final ITypeTraits[] typeTraits;
- private final boolean isDelete;
+ private final boolean isAntimatter;
- public LSMRTreeTupleWriterFactory(ITypeTraits[] typeTraits, boolean isDelete) {
+ public LSMRTreeTupleWriterFactory(ITypeTraits[] typeTraits, boolean isAntimatter) {
super(typeTraits);
this.typeTraits = typeTraits;
- this.isDelete = isDelete;
+ this.isAntimatter = isAntimatter;
}
@Override
- public ITreeIndexTupleWriter createTupleWriter() {
- return new LSMRTreeTupleWriter(typeTraits, isDelete);
+ public ILSMTreeTupleWriter createTupleWriter() {
+ return new LSMRTreeTupleWriter(typeTraits, isAntimatter);
}
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
index c18823a..eec6f3b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
@@ -20,8 +20,8 @@
package org.apache.hyracks.storage.am.lsm.rtree.tuples;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
public class LSMRTreeTupleWriterFactoryForPointMBR extends TypeAwareTupleWriterFactory {
@@ -29,17 +29,21 @@
private final int keyFieldCount;
private final int valueFieldCount;
private final boolean antimatterAware;
+ private final boolean isAntimatter;
public LSMRTreeTupleWriterFactoryForPointMBR(ITypeTraits[] typeTraits, int keyFieldCount, int valueFieldCount,
- boolean antimatterAware) {
+ boolean antimatterAware, boolean isDelete) {
super(typeTraits);
this.keyFieldCount = keyFieldCount;
this.valueFieldCount = valueFieldCount;
this.antimatterAware = antimatterAware;
+ this.isAntimatter = isDelete;
}
@Override
- public ITreeIndexTupleWriter createTupleWriter() {
- return new LSMRTreeTupleWriterForPointMBR(typeTraits, keyFieldCount, valueFieldCount, antimatterAware);
+ public ILSMTreeTupleWriter createTupleWriter() {
+ return new LSMRTreeTupleWriterForPointMBR(typeTraits, keyFieldCount, valueFieldCount, antimatterAware,
+ isAntimatter);
}
+
}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
index 5de4d20..4ccfd47 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
import org.apache.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
@@ -40,16 +41,17 @@
* the stored point MBR [0.4, 0.3, 1] and generates a tuple reference which is externally shown as [0.4, 0.3, 0.4, 0.3, 1].
*/
-public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter {
+public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter implements ILSMTreeTupleWriter {
private final int inputKeyFieldCount; //double field count for mbr secondary key of an input tuple
private final int valueFieldCount; //value(or payload or primary key) field count (same for an input tuple and a stored tuple)
private final int inputTotalFieldCount; //total field count (key + value fields) of an input tuple.
private final int storedKeyFieldCount; //double field count to be stored for the mbr secondary key
private final int storedTotalFieldCount; //total field count (key + value fields) of a stored tuple.
private final boolean antimatterAware;
+ private boolean isAntimatter;
public LSMRTreeTupleWriterForPointMBR(ITypeTraits[] typeTraits, int keyFieldCount, int valueFieldCount,
- boolean antimatterAware) {
+ boolean antimatterAware, boolean isAntimatter) {
super(typeTraits);
this.inputKeyFieldCount = keyFieldCount;
this.valueFieldCount = valueFieldCount;
@@ -57,6 +59,7 @@
this.storedKeyFieldCount = keyFieldCount / 2;
this.storedTotalFieldCount = storedKeyFieldCount + valueFieldCount;
this.antimatterAware = antimatterAware;
+ this.isAntimatter = isAntimatter;
}
@Override
@@ -87,7 +90,7 @@
targetBuf[runner++] = (byte) 0;
}
- // write field slots for variable length fields which applies only to value fields in RTree
+ // write field slots for variable length fields which applies only to value fields in RTree
for (int i = inputKeyFieldCount; i < inputTotalFieldCount; i++) {
if (!typeTraits[i].isFixedLength()) {
runner += VarLenIntEncoderDecoder.encode(tuple.getFieldLength(i), targetBuf, runner);
@@ -106,12 +109,19 @@
}
//set antimatter bit if necessary
+ //this is used when we flush an in-memory rtree into disk
+ //and insert anti-matter tuples from in-memory buddy btree into disk rtree
if (antimatterAware) {
if (tuple instanceof ILSMTreeTupleReference && ((ILSMTreeTupleReference) tuple).isAntimatter()) {
setAntimatterBit(targetBuf, targetOff);
}
}
+ //this is used during creating secondary index operation, where we explicitly insert some antimatter tuple
+ if (isAntimatter) {
+ setAntimatterBit(targetBuf, targetOff);
+ }
+
return runner - targetOff;
}
@@ -123,10 +133,12 @@
"writeTupleFields(ITupleReference, int, int, byte[], int) not implemented for RTreeTypeAwareTupleWriterForPointMBR class.");
}
+ @Override
protected int getNullFlagsBytes(ITupleReference tuple) {
- return (int) Math.ceil((double) (storedTotalFieldCount + (antimatterAware ? 1 : 0)) / 8.0);
+ return (int) Math.ceil((storedTotalFieldCount + (antimatterAware ? 1 : 0)) / 8.0);
}
+ @Override
protected int getFieldSlotsBytes(ITupleReference tuple) {
int fieldSlotBytes = 0;
for (int i = inputKeyFieldCount; i < inputTotalFieldCount; i++) {
@@ -146,4 +158,9 @@
// Set leftmost bit to 1.
targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
}
+
+ @Override
+ public void setAntimatter(boolean isAntimatter) {
+ this.isAntimatter = isAntimatter;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java
index ac4f3c8..31a9278 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java
@@ -28,17 +28,17 @@
private static final long serialVersionUID = 1L;
private ITypeTraits[] typeTraits;
- private final boolean isDelete;
+ private final boolean isAntimatter;
- public LSMTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits, boolean isDelete) {
+ public LSMTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits, boolean isAntimatter) {
super(typeTraits);
this.typeTraits = typeTraits;
- this.isDelete = isDelete;
+ this.isAntimatter = isAntimatter;
}
@Override
public ITreeIndexTupleWriter createTupleWriter() {
- if (isDelete) {
+ if (isAntimatter) {
return new TypeAwareTupleWriter(typeTraits);
} else {
return new RTreeTypeAwareTupleWriter(typeTraits);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index d390be0..e37fa4a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -92,7 +92,7 @@
ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
if (isPointMBR) {
rtreeLeafFrameTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false);
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false, false);
} else {
rtreeLeafFrameTupleWriterFactory = rtreeInteriorFrameTupleWriterFactory;
}
@@ -154,24 +154,30 @@
new LSMRTreeTupleWriterFactory(typeTraits, false);
ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory;
ITreeIndexTupleWriterFactory rtreeLeafFrameCopyTupleWriterFactory;
+ ITreeIndexTupleWriterFactory rtreeLeafFrameBulkLoadWriterFactory;
if (isPointMBR) {
int keyFieldCount = rtreeCmpFactories.length;
int valueFieldCount = btreeComparatorFactories.length - keyFieldCount;
rtreeLeafFrameTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true);
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
rtreeLeafFrameCopyTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true);
-
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
+ rtreeLeafFrameBulkLoadWriterFactory =
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
} else {
rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
rtreeLeafFrameCopyTupleWriterFactory = new LSMRTreeCopyTupleWriterFactory(typeTraits);
+ rtreeLeafFrameBulkLoadWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
}
+
LSMRTreeTupleWriterFactory btreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, true);
ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(
rtreeInteriorFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory,
valueProviderFactories, rtreePolicyType, isPointMBR);
+ ITreeIndexFrameFactory rtreeLeafFrameBulkLoadFactory = new RTreeNSMLeafFrameFactory(
+ rtreeLeafFrameBulkLoadWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
@@ -183,7 +189,7 @@
typeTraits.length, isPointMBR);
TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
- freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
+ freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameBulkLoadFactory, rtreeCmpFactories,
typeTraits.length, isPointMBR);
// The first field is for the sorted curve (e.g. Hilbert curve), and the
@@ -242,7 +248,7 @@
ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
if (isPointMBR) {
rtreeLeafFrameTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false);
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false, false);
} else {
rtreeLeafFrameTupleWriterFactory = rtreeInteriorFrameTupleWriterFactory;
}