ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set
-Fixed a bug that FLUSH_LSN for flushed disk components is not
correctly set (not increasing) when an NC has multiple partitions.
-Added LSMIOOperationCallback unit tests to cover this bug
Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1771
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index f903b65..d04443a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -107,8 +107,14 @@
return pointable.getLength() == 0 ? INVALID : pointable.longValue();
}
- public void updateLastLSN(long lastLSN) {
- mutableLastLSNs[writeIndex] = lastLSN;
+ public synchronized void updateLastLSN(long lastLSN) {
+ if (!flushRequested[writeIndex]) {
+ //if the memory component pointed by writeIndex is being flushed, we should ignore this update call
+ //since otherwise the original LSN is overwritten.
+ //Moreover, since the memory component is already being flushed, the next scheduleFlush request must fail.
+ //See https://issues.apache.org/jira/browse/ASTERIXDB-1917
+ mutableLastLSNs[writeIndex] = lastLSN;
+ }
}
public void setFirstLSN(long firstLSN) {
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
new file mode 100644
index 0000000..bc206e2
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMBTreeIOOperationCallbackTest extends TestCase {
+
+ public void testNormalSequence() {
+ try {
+ LSMBTreeIOOperationCallback callback = new LSMBTreeIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ public void testOverWrittenLSN() {
+ try {
+ LSMBTreeIOOperationCallback callback = new LSMBTreeIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush first component again
+ //this call should fail
+ callback.updateLastLSN(3);
+ //there is no corresponding beforeOperation, since the first component is being flush
+ //the scheduleFlush request would fail this time
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java
new file mode 100644
index 0000000..fe84d4a
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMBTreeWithBuddyIOOperationCallbackTest extends TestCase {
+
+ public void testNormalSequence() {
+ try {
+ LSMBTreeWithBuddyIOOperationCallback callback = new LSMBTreeWithBuddyIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ public void testOverWrittenLSN() {
+ try {
+ LSMBTreeWithBuddyIOOperationCallback callback = new LSMBTreeWithBuddyIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush first component again
+ //this call should fail
+ callback.updateLastLSN(3);
+ //there is no corresponding beforeOperation, since the first component is being flush
+ //the scheduleFlush request would fail this time
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java
new file mode 100644
index 0000000..1f99db3
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMInvertedIndexIOOperationCallbackTest extends TestCase {
+
+ public void testNormalSequence() {
+ try {
+ LSMInvertedIndexIOOperationCallback callback = new LSMInvertedIndexIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ public void testOverWrittenLSN() {
+ try {
+ LSMInvertedIndexIOOperationCallback callback = new LSMInvertedIndexIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush first component again
+ //this call should fail
+ callback.updateLastLSN(3);
+ //there is no corresponding beforeOperation, since the first component is being flush
+ //the scheduleFlush request would fail this time
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java
new file mode 100644
index 0000000..38f7ee9
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMRTreeIOOperationCallbackTest extends TestCase {
+
+ public void testNormalSequence() {
+ try {
+ LSMRTreeIOOperationCallback callback = new LSMRTreeIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ public void testOverWrittenLSN() {
+ try {
+ LSMRTreeIOOperationCallback callback = new LSMRTreeIOOperationCallback();
+ callback.setNumOfMutableComponents(2);
+
+ //request to flush first component
+ callback.updateLastLSN(1);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush second component
+ callback.updateLastLSN(2);
+ callback.beforeOperation(LSMOperationType.FLUSH);
+
+ //request to flush first component again
+ //this call should fail
+ callback.updateLastLSN(3);
+ //there is no corresponding beforeOperation, since the first component is being flush
+ //the scheduleFlush request would fail this time
+
+ Assert.assertEquals(1, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+ Assert.assertEquals(2, callback.getComponentLSN(null));
+ callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+}