Change the API for writing and reading metadata pages
Change-Id: Iadad522ab5568677aa816c74fc1d63acad505380
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1396
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
index da7c397..43e1ff3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
@@ -16,17 +16,14 @@
! specific language governing permissions and limitations
! under the License.
!-->
-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>hyracks-storage-am-lsm-common</artifactId>
-
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
<version>0.2.18-SNAPSHOT</version>
</parent>
-
<licenses>
<license>
<name>Apache License, Version 2.0</name>
@@ -35,11 +32,9 @@
<comments>A business-friendly OSS license</comments>
</license>
</licenses>
-
<properties>
<root.dir>${basedir}/../..</root.dir>
</properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
@@ -76,5 +71,10 @@
<artifactId>hyracks-dataflow-std</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
index cf77f30..1c853c8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
@@ -19,5 +19,5 @@
package org.apache.hyracks.storage.am.lsm.common.api;
public interface ILSMComponentFilterFrameFactory {
- public ILSMComponentFilterFrame createFrame();
+ public ILSMComponentFilterReference createFrame();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
index 20598ca..12cafef 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
@@ -22,9 +22,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex.AbstractTreeIndexBulkLoader;
public interface ILSMComponentFilterManager {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterReference.java
similarity index 66%
rename from hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrame.java
rename to hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterReference.java
index 4c159b7..5e6f4eb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterReference.java
@@ -18,27 +18,23 @@
*/
package org.apache.hyracks.storage.am.lsm.common.api;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-public interface ILSMComponentFilterFrame {
+public interface ILSMComponentFilterReference extends IValueReference, IPointable {
- public ICachedPage getPage();
+ void writeMinTuple(ITupleReference tuple);
- public void setPage(ICachedPage page);
+ void writeMaxTuple(ITupleReference tuple);
- public void writeMinTuple(ITupleReference tuple);
+ boolean isMinTupleSet();
- public void writeMaxTuple(ITupleReference tuple);
+ boolean isMaxTupleSet();
- public void initBuffer();
+ ITupleReference getMinTuple();
- public boolean isMinTupleSet();
+ ITupleReference getMaxTuple();
- public boolean isMaxTupleSet();
-
- public ITupleReference getMinTuple();
-
- public ITupleReference getMaxTuple();
-
+ void reset();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrame.java
deleted file mode 100644
index 5d74da1..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrame.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common.frames;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrame;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-
-public class LSMComponentFilterFrame implements ILSMComponentFilterFrame {
-
- // This page consists of two tuples that represents the minimum and maximum tuples in an LSM component.
-
- // A-one byte to indicate whether the filter tuples were set yet.
- private static final int minTupleIsSetIndicatorOff = 0;
- private static final int maxTupleIsSetIndicatorOff = 1;
-
- private final int minTupleOff;
- private final int maxTupleOff;
-
- private final ITreeIndexTupleWriter tupleWriter;
-
- protected ICachedPage page = null;
- protected ByteBuffer buf = null;
-
- private ITreeIndexTupleReference minTuple;
- private ITreeIndexTupleReference maxTuple;
-
- public LSMComponentFilterFrame(ITreeIndexTupleWriter tupleWriter, int pageSize) {
- this.tupleWriter = tupleWriter;
- this.minTupleOff = maxTupleIsSetIndicatorOff + 1;
- this.maxTupleOff = maxTupleIsSetIndicatorOff + 1 + (pageSize / 2);
-
- this.minTuple = tupleWriter.createTupleReference();
- this.maxTuple = tupleWriter.createTupleReference();
- }
-
- @Override
- public void initBuffer() {
- buf.put(minTupleIsSetIndicatorOff, (byte) 0);
- buf.put(maxTupleIsSetIndicatorOff, (byte) 0);
- }
-
- @Override
- public ICachedPage getPage() {
- return page;
- }
-
- @Override
- public void setPage(ICachedPage page) {
- this.page = page;
- this.buf = page.getBuffer();
- }
-
- @Override
- public void writeMinTuple(ITupleReference tuple) {
- tupleWriter.writeTuple(tuple, buf.array(), minTupleOff);
- buf.put(minTupleIsSetIndicatorOff, (byte) 1);
- }
-
- @Override
- public void writeMaxTuple(ITupleReference tuple) {
- tupleWriter.writeTuple(tuple, buf.array(), maxTupleOff);
- buf.put(maxTupleIsSetIndicatorOff, (byte) 1);
- }
-
- @Override
- public boolean isMinTupleSet() {
- return buf.get(minTupleIsSetIndicatorOff) == (byte) 1 ? true : false;
- }
-
- @Override
- public boolean isMaxTupleSet() {
- return buf.get(maxTupleIsSetIndicatorOff) == (byte) 1 ? true : false;
- }
-
- @Override
- public ITupleReference getMinTuple() {
- minTuple.resetByTupleOffset(buf, minTupleOff);
- return minTuple;
- }
-
- @Override
- public ITupleReference getMaxTuple() {
- maxTuple.resetByTupleOffset(buf, maxTupleOff);
- return maxTuple;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
index 7a75534..999f72b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
@@ -20,21 +20,19 @@
package org.apache.hyracks.storage.am.lsm.common.frames;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrame;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterReference;
public class LSMComponentFilterFrameFactory implements ILSMComponentFilterFrameFactory {
private final ITreeIndexTupleWriterFactory tupleWriterFactory;
- private final int pageSize;
- public LSMComponentFilterFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, int pageSize) {
+ public LSMComponentFilterFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
this.tupleWriterFactory = tupleWriterFactory;
- this.pageSize = pageSize;
}
@Override
- public ILSMComponentFilterFrame createFrame() {
- return new LSMComponentFilterFrame(tupleWriterFactory.createTupleWriter(), pageSize);
+ public ILSMComponentFilterReference createFrame() {
+ return new LSMComponentFilterReference(tupleWriterFactory.createTupleWriter());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterReference.java
new file mode 100644
index 0000000..f04cbb0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterReference.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.frames;
+
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.BooleanPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterReference;
+
+/**
+ * When written to disk:
+ * [min set?][max set?][min length][min][max length][max]
+ */
+public class LSMComponentFilterReference implements ILSMComponentFilterReference {
+
+ private static final int MIN_SET_INDICATOR_OFFSET = 0;
+ private static final int MAX_SET_INDICATOR_OFFSET = 1;
+
+ private final ArrayBackedValueStorage min;
+ private final ArrayBackedValueStorage max;
+ private ArrayBackedValueStorage binaryFilter;
+ private final ITreeIndexTupleWriter tupleWriter;
+ private ITreeIndexTupleReference minTuple;
+ private ITreeIndexTupleReference maxTuple;
+
+ public LSMComponentFilterReference(ITreeIndexTupleWriter tupleWriter) {
+ this.tupleWriter = tupleWriter;
+ min = new ArrayBackedValueStorage();
+ max = new ArrayBackedValueStorage();
+ binaryFilter = new ArrayBackedValueStorage();
+ minTuple = tupleWriter.createTupleReference();
+ maxTuple = tupleWriter.createTupleReference();
+ }
+
+ @Override
+ public void writeMinTuple(ITupleReference tuple) {
+ binaryFilter.reset();
+ min.reset();
+ min.setSize(tupleWriter.bytesRequired(tuple));
+ tupleWriter.writeTuple(tuple, min.getByteArray(), 0);
+ }
+
+ @Override
+ public void writeMaxTuple(ITupleReference tuple) {
+ binaryFilter.reset();
+ max.reset();
+ max.setSize(tupleWriter.bytesRequired(tuple));
+ tupleWriter.writeTuple(tuple, max.getByteArray(), 0);
+ }
+
+ @Override
+ public boolean isMinTupleSet() {
+ return min.getLength() > 0;
+ }
+
+ @Override
+ public boolean isMaxTupleSet() {
+ return max.getLength() > 0;
+ }
+
+ @Override
+ public ITupleReference getMinTuple() {
+ minTuple.resetByTupleOffset(min.getByteArray(), 0);
+ return minTuple;
+ }
+
+ @Override
+ public ITupleReference getMaxTuple() {
+ maxTuple.resetByTupleOffset(max.getByteArray(), 0);
+ return maxTuple;
+ }
+
+ @Override
+ public byte[] getByteArray() {
+ if (binaryFilter.getLength() == 0) {
+ int binarySize = 2;
+ if (min.getLength() > 0) {
+ binarySize += Integer.BYTES + min.getLength();
+ }
+ if (max.getLength() > 0) {
+ binarySize += Integer.BYTES + max.getLength();
+ }
+ binaryFilter.setSize(binarySize);
+ byte[] buf = binaryFilter.getByteArray();
+ BooleanPointable.setBoolean(buf, MIN_SET_INDICATOR_OFFSET, min.getLength() == 0);
+ BooleanPointable.setBoolean(buf, MAX_SET_INDICATOR_OFFSET, max.getLength() == 0);
+ int offset = 2;
+ if (min.getLength() > 0) {
+ IntegerPointable.setInteger(buf, offset, min.getLength());
+ offset += Integer.BYTES;
+ System.arraycopy(min.getByteArray(), 0, buf, offset, min.getLength());
+ }
+ if (max.getLength() > 0) {
+ IntegerPointable.setInteger(buf, offset, max.getLength());
+ offset += Integer.BYTES;
+ System.arraycopy(max.getByteArray(), 0, buf, offset, max.getLength());
+ }
+ }
+ return binaryFilter.getByteArray();
+ }
+
+ @Override
+ public int getStartOffset() {
+ return 0;
+ }
+
+ @Override
+ public int getLength() {
+ return binaryFilter.getLength();
+ }
+
+ @Override
+ public void reset() {
+ min.reset();
+ max.reset();
+ binaryFilter.reset();
+ }
+
+ @Override
+ public void set(byte[] bytes, int start, int length) {
+ boolean isMinSet = BooleanPointable.getBoolean(bytes, MIN_SET_INDICATOR_OFFSET + start);
+ boolean isMaxSet = BooleanPointable.getBoolean(bytes, MAX_SET_INDICATOR_OFFSET + start);
+ int srcOffset = start + 2;
+ if (isMinSet) {
+ min.setSize(IntegerPointable.getInteger(bytes, srcOffset));
+ srcOffset += Integer.BYTES;
+ System.arraycopy(bytes, srcOffset, min.getByteArray(), 0, min.getLength());
+ srcOffset += min.getLength();
+ }
+ if (isMaxSet) {
+ max.setSize(IntegerPointable.getInteger(bytes, srcOffset));
+ srcOffset += Integer.BYTES;
+ System.arraycopy(bytes, srcOffset, max.getByteArray(), 0, max.getLength());
+ }
+ binaryFilter.reset();
+ }
+
+ @Override
+ public void set(IValueReference pointer) {
+ set(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
index 5771728..08c75dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
@@ -25,7 +25,7 @@
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
@@ -44,34 +44,34 @@
}
@Override
- public int takePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ public int takePage(ITreeIndexMetadataFrame metaFrame) throws HyracksDataException {
// The very first call returns page id 2 because the BTree uses
// the first page as metadata page, and the second page as root page.
return currentPageId.incrementAndGet();
}
@Override
- public int takeBlock(ITreeIndexMetaDataFrame metaFrame, int count) throws HyracksDataException {
+ public int takeBlock(ITreeIndexMetadataFrame metaFrame, int count) throws HyracksDataException {
return currentPageId.getAndUpdate(operand -> operand + count) + 1;
}
@Override
- public int getMaxPageId(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ public int getMaxPageId(ITreeIndexMetadataFrame metaFrame) throws HyracksDataException {
return currentPageId.get();
}
@Override
- public ITreeIndexMetaDataFrame createMetadataFrame() {
+ public ITreeIndexMetadataFrame createMetadataFrame() {
return null;
}
@Override
- public void releasePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
+ public void releasePage(ITreeIndexMetadataFrame metaFrame, int freePage) throws HyracksDataException {
throw new HyracksDataException("Pages of an in memory index are released through the virtual buffer cache");
}
@Override
- public void releaseBlock(ITreeIndexMetaDataFrame metaFrame, int startingPage, int count)
+ public void releaseBlock(ITreeIndexMetadataFrame metaFrame, int startingPage, int count)
throws HyracksDataException {
throw new HyracksDataException("Pages of an in memory index are released through the virtual buffer cache");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
index c678878..cf50c00 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
@@ -18,15 +18,21 @@
*/
package org.apache.hyracks.storage.am.lsm.common.impls;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
public abstract class AbstractLSMComponent implements ILSMComponent {
+ public static final MutableArrayValueReference MARKER_LSN_KEY = new MutableArrayValueReference("Marker"
+ .getBytes());
+
protected ComponentState state;
protected int readerCount;
protected final ILSMComponentFilter filter;
protected long mostRecentMarkerLSN;
+ protected final LongPointable pointable = (LongPointable) LongPointable.FACTORY.createPointable();
public AbstractLSMComponent(ILSMComponentFilter filter, long mostRecentMarkerLSN) {
this.filter = filter;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 43b6f92..731d312 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -37,7 +37,8 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -94,7 +95,7 @@
if (metadataPage < 0) {
return TreeIndexState.INVALID;
}
- ITreeIndexMetaDataFrame metadataFrame = treeIndex.getPageManager().createMetadataFrame();
+ ITreeIndexMetadataFrame metadataFrame = treeIndex.getPageManager().createMetadataFrame();
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(treeIndex.getFileId(), metadataPage),
false);
page.acquireReadLatch();
@@ -102,7 +103,7 @@
metadataFrame.setPage(page);
if (!metadataFrame.isValid()) {
return TreeIndexState.INVALID;
- } else if (metadataFrame.getVersion() != ITreeIndexMetaDataFrame.VERSION) {
+ } else if (metadataFrame.getVersion() != ITreeIndexFrame.Constants.VERSION) {
return TreeIndexState.VERSION_MISMATCH;
} else {
return TreeIndexState.VALID;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilter.java
index f7bc2ad..32937a9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilter.java
@@ -70,7 +70,7 @@
tupleWriter.writeTuple(tuple, minTupleBytes, 0);
minTupleBuf = ByteBuffer.wrap(minTupleBytes);
minTuple = tupleWriter.createTupleReference();
- ((ITreeIndexTupleReference) minTuple).resetByTupleOffset(minTupleBuf, 0);
+ ((ITreeIndexTupleReference) minTuple).resetByTupleOffset(minTupleBuf.array(), 0);
} else {
int c = cmp.compare(tuple, minTuple);
if (c < 0) {
@@ -82,7 +82,7 @@
} else {
tupleWriter.writeTuple(tuple, minTupleBytes, 0);
}
- ((ITreeIndexTupleReference) minTuple).resetByTupleOffset(minTupleBuf, 0);
+ ((ITreeIndexTupleReference) minTuple).resetByTupleOffset(minTupleBuf.array(), 0);
}
}
if (maxTuple == null) {
@@ -91,7 +91,7 @@
tupleWriter.writeTuple(tuple, maxTupleBytes, 0);
maxTupleBuf = ByteBuffer.wrap(maxTupleBytes);
maxTuple = tupleWriter.createTupleReference();
- ((ITreeIndexTupleReference) maxTuple).resetByTupleOffset(maxTupleBuf, 0);
+ ((ITreeIndexTupleReference) maxTuple).resetByTupleOffset(maxTupleBuf.array(), 0);
} else {
int c = cmp.compare(tuple, maxTuple);
if (c > 0) {
@@ -103,7 +103,7 @@
} else {
tupleWriter.writeTuple(tuple, maxTupleBytes, 0);
}
- ((ITreeIndexTupleReference) maxTuple).resetByTupleOffset(maxTupleBuf, 0);
+ ((ITreeIndexTupleReference) maxTuple).resetByTupleOffset(maxTupleBuf.array(), 0);
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
index 0fbbbdf..a8b4c70 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
@@ -25,22 +25,19 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrame;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterReference;
public class LSMComponentFilterManager implements ILSMComponentFilterManager {
- private final IBufferCache bufferCache;
+ public static final MutableArrayValueReference FILTER_KEY = new MutableArrayValueReference("Filter".getBytes());
private final ILSMComponentFilterFrameFactory filterFrameFactory;
- public LSMComponentFilterManager(IBufferCache bufferCache, ILSMComponentFilterFrameFactory filterFrameFactory) {
- this.bufferCache = bufferCache;
+ public LSMComponentFilterManager(ILSMComponentFilterFrameFactory filterFrameFactory) {
this.filterFrameFactory = filterFrameFactory;
}
@@ -56,11 +53,8 @@
@Override
public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
IMetadataPageManager treeMetaManager = (IMetadataPageManager) treeIndex.getPageManager();
- ICachedPage filterPage = treeMetaManager.getFilterPage();
+ ILSMComponentFilterReference filterFrame = filterFrameFactory.createFrame();
try {
- ILSMComponentFilterFrame filterFrame = filterFrameFactory.createFrame();
- filterFrame.setPage(filterPage);
- filterFrame.initBuffer();
if (filter.getMinTuple() != null) {
filterFrame.writeMinTuple(filter.getMinTuple());
}
@@ -68,39 +62,23 @@
filterFrame.writeMaxTuple(filter.getMaxTuple());
}
} finally {
- treeMetaManager.setFilterPage(filterPage);
+ treeMetaManager.put(treeMetaManager.createMetadataFrame(), FILTER_KEY, filterFrame);
}
}
@Override
public boolean readFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
- int fileId = treeIndex.getFileId();
IMetadataPageManager treeMetaManager = (IMetadataPageManager) treeIndex.getPageManager();
- int componentFilterPageId = treeMetaManager.getFilterPageId();
- if (componentFilterPageId < 0) {
+ ILSMComponentFilterReference filterFrame = filterFrameFactory.createFrame();
+ treeMetaManager.get(treeMetaManager.createMetadataFrame(), FILTER_KEY, filterFrame);
+ // TODO: Filters never have one of min/max set and the other not
+ if (!filterFrame.isMinTupleSet() || !filterFrame.isMaxTupleSet()) {
return false;
}
-
- ICachedPage filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, componentFilterPageId),
- false);
-
- filterPage.acquireReadLatch();
- try {
- ILSMComponentFilterFrame filterFrame = filterFrameFactory.createFrame();
- filterFrame.setPage(filterPage);
-
- if (!filterFrame.isMinTupleSet() || !filterFrame.isMaxTupleSet()) {
- return false;
- }
- List<ITupleReference> filterTuples = new ArrayList<>();
- filterTuples.add(filterFrame.getMinTuple());
- filterTuples.add(filterFrame.getMaxTuple());
- updateFilterInfo(filter, filterTuples);
-
- } finally {
- filterPage.releaseReadLatch();
- bufferCache.unpin(filterPage);
- }
+ List<ITupleReference> filterTuples = new ArrayList<>();
+ filterTuples.add(filterFrame.getMinTuple());
+ filterTuples.add(filterFrame.getMaxTuple());
+ updateFilterInfo(filter, filterTuples);
return true;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 896d513..a6868f0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -66,7 +66,7 @@
//only durable indexes are replicated
this.replicationEnabled = replicationEnabled && lsmIndex.isDurable();
if (replicationEnabled) {
- this.componentsToBeReplicated = new ArrayList<ILSMComponent>();
+ this.componentsToBeReplicated = new ArrayList<>();
}
}
@@ -314,7 +314,7 @@
for (ILSMComponent inactiveComp : inactiveDiskComponents) {
if (((AbstractDiskLSMComponent) inactiveComp).getFileReferenceCount() == 1) {
if (inactiveDiskComponentsToBeDeleted == null) {
- inactiveDiskComponentsToBeDeleted = new LinkedList<ILSMComponent>();
+ inactiveDiskComponentsToBeDeleted = new LinkedList<>();
}
inactiveDiskComponentsToBeDeleted.add(inactiveComp);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index 416f859..49f9060 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -22,9 +22,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -36,7 +34,7 @@
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
-import org.apache.hyracks.storage.common.buffercache.IQueueInfo;
+import org.apache.hyracks.storage.common.buffercache.VirtualPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapManager;
import org.apache.hyracks.storage.common.file.TransientFileMapManager;
@@ -106,19 +104,19 @@
VirtualPage prev = null;
VirtualPage curr = bucket.cachedPage;
while (curr != null) {
- if (BufferedFileHandle.getFileId(curr.dpid) == fileId) {
+ if (BufferedFileHandle.getFileId(curr.dpid()) == fileId) {
if (prev == null) {
- bucket.cachedPage = curr.next;
+ bucket.cachedPage = curr.next();
curr.reset();
curr = bucket.cachedPage;
} else {
- prev.next = curr.next;
+ prev.next(curr.next());
curr.reset();
- curr = prev.next;
+ curr = prev.next();
}
} else {
prev = curr;
- curr = curr.next;
+ curr = curr.next();
}
}
} finally {
@@ -134,18 +132,18 @@
int end = nextFree - 1;
while (start < end) {
VirtualPage lastUsed = pages.get(end);
- while (end > 0 && lastUsed.dpid == -1) {
+ while (end > 0 && lastUsed.dpid() == -1) {
--end;
lastUsed = pages.get(end);
}
if (end == 0) {
- nextFree = lastUsed.dpid == -1 ? 0 : 1;
+ nextFree = lastUsed.dpid() == -1 ? 0 : 1;
break;
}
VirtualPage firstUnused = pages.get(start);
- while (start < end && firstUnused.dpid != -1) {
+ while (start < end && firstUnused.dpid() != -1) {
++start;
firstUnused = pages.get(start);
}
@@ -177,10 +175,10 @@
try {
page = bucket.cachedPage;
while (page != null) {
- if (page.dpid == dpid) {
+ if (page.dpid() == dpid) {
return page;
}
- page = page.next;
+ page = page.next();
}
if (!newPage) {
@@ -189,7 +187,7 @@
}
page = getOrAllocPage(dpid);
- page.next = bucket.cachedPage;
+ page.next(bucket.cachedPage);
bucket.cachedPage = page;
} finally {
bucket.bucketLock.unlock();
@@ -207,14 +205,14 @@
VirtualPage page;
synchronized (pages) {
if (nextFree >= pages.size()) {
- page = new VirtualPage(allocator.allocate(pageSize, 1)[0]);
- page.multiplier = 1;
+ page = new VirtualPage(allocator.allocate(pageSize, 1)[0], pageSize);
+ page.multiplier(1);
pages.add(page);
} else {
page = pages.get(nextFree);
}
++nextFree;
- page.dpid = dpid;
+ page.dpid(dpid);
}
return page;
}
@@ -248,8 +246,8 @@
} else {
largePages.getAndAdd(multiplier - origMultiplier);
}
- ((VirtualPage) cPage).buffer = newBuffer;
- ((VirtualPage) cPage).multiplier = multiplier;
+ ((VirtualPage) cPage).buffer(newBuffer);
+ ((VirtualPage) cPage).multiplier(multiplier);
}
@Override
@@ -352,76 +350,6 @@
}
}
- private class VirtualPage implements ICachedPage {
- ByteBuffer buffer;
- final ReadWriteLock latch;
- volatile long dpid;
- int multiplier;
- VirtualPage next;
-
- public VirtualPage(ByteBuffer buffer) {
- this.buffer = buffer;
- latch = new ReentrantReadWriteLock(true);
- dpid = -1;
- next = null;
- }
-
- public void reset() {
- dpid = -1;
- next = null;
- }
-
- @Override
- public ByteBuffer getBuffer() {
- return buffer;
- }
-
- @Override
- public void acquireReadLatch() {
- latch.readLock().lock();
- }
-
- @Override
- public void releaseReadLatch() {
- latch.readLock().unlock();
- }
-
- @Override
- public void acquireWriteLatch() {
- latch.writeLock().lock();
- }
-
- @Override
- public void releaseWriteLatch(boolean markDirty) {
- latch.writeLock().unlock();
- }
-
- @Override
- public boolean confiscated() {
- return false;
- }
-
- @Override
- public IQueueInfo getQueueInfo() {
- return null;
- }
-
- @Override
- public void setQueueInfo(IQueueInfo queueInfo) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int getPageSize() {
- return pageSize;
- }
-
- @Override
- public int getFrameSizeMultiplier() {
- return multiplier;
- }
- }
-
//These 4 methods aren't applicable here.
@Override
public int createMemFile() throws HyracksDataException {