Make LSN recording in components compatible with append-only
Change-Id: I5610e903be3347893e676c915b98316b06073c25
Reviewed-on: https://asterix-gerrit.ics.uci.edu/265
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
new file mode 100644
index 0000000..6365678
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Comparison of dataset scan results to secondary index point lookups for BTree and RTree
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type TwitterUserType as {
+ screen-name: string,
+ lang: string,
+ friends_count: int32,
+ statuses_count: int32,
+ name: string,
+ followers_count: int32
+}
+
+create type TweetMessageType as {
+ tweetid: int64,
+ user: string,
+ sender-location: point,
+ send-time: datetime,
+ forward-from: int64,
+ retweet-from: int64,
+ referred-topics: {{ string }},
+ message-text: string
+}
+
+create type FacebookMessageType as {
+ message-id: int64,
+ author-id: int64,
+ in-response-to: int64?,
+ sender-location: point,
+ send-time: datetime,
+ message: string
+}
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+create dataset FacebookMessages(FacebookMessageType) primary key message-id;
+
+create index fbmAutIdIx on FacebookMessages(author-id);
+
+create index twmSndTmIx on TweetMessages(send-time);
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
new file mode 100644
index 0000000..d1f883d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description : Index Nested Loop Join on three datasets. Two index nested loop joins should be nested properly.
+ * Success : Yes
+ */
+ use dataverse test;
+
+load dataset TweetMessages
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/index-join/tw_messages.adm"),("format"="adm"));
+
+load dataset FacebookMessages
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/fbm-with-send-time.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
new file mode 100644
index 0000000..be4ed69
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description : BTree selection compared with dataset scan
+ * Success : Yes
+ */
+use dataverse test;
+
+let $a := (for $n in dataset FacebookMessages return $n)
+
+let $b := (for $n in dataset FacebookMessages where $n.author-id >= 0 and
+$n.author-id <= 10 return $n
+)
+
+for $x in $a
+where every $y in $b satisfies $y.message-id != $x.message-id
+return $x;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
new file mode 100644
index 0000000..b161db6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description : BTree selection compared with dataset scan
+ * Success : Yes
+ */
+use dataverse test;
+
+let $a :=count (
+let $pts := for $t in dataset TweetMessages return $t.sender-location
+
+for $t in dataset TweetMessages
+for $p in $pts
+where spatial-intersect($t.sender-location,$p)
+return $t
+)
+
+let $b := count(
+for $t in dataset TweetMessages return $t
+)
+
+
+return $a = $b
diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
new file mode 100644
index 0000000..27ba77d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
@@ -0,0 +1 @@
+true
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java b/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index 76a11d1..b3cae06 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -24,12 +24,9 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
// A single LSMIOOperationCallback per LSM index used to perform actions around Flush and Merge operations
@@ -96,35 +93,11 @@
protected void putLSNIntoMetadata(ITreeIndex treeIndex, List<ILSMComponent> oldComponents)
throws HyracksDataException {
long componentLSN = getComponentLSN(oldComponents);
- int fileId = treeIndex.getFileId();
- IBufferCache bufferCache = treeIndex.getBufferCache();
- ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
- int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
- ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, metadataPageId), false);
- metadataPage.acquireWriteLatch();
- try {
- metadataFrame.setPage(metadataPage);
- metadataFrame.setLSN(componentLSN);
- } finally {
- metadataPage.releaseWriteLatch(true);
- bufferCache.unpin(metadataPage);
- }
+ treeIndex.getMetaManager().setLSN(componentLSN);
}
protected long getTreeIndexLSN(ITreeIndex treeIndex) throws HyracksDataException {
- int fileId = treeIndex.getFileId();
- IBufferCache bufferCache = treeIndex.getBufferCache();
- ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
- int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
- ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, metadataPageId), false);
- metadataPage.acquireReadLatch();
- try {
- metadataFrame.setPage(metadataPage);
- return metadataFrame.getLSN();
- } finally {
- metadataPage.releaseReadLatch();
- bufferCache.unpin(metadataPage);
- }
+ return treeIndex.getMetaManager().getLSN();
}
public void updateLastLSN(long lastLSN) {
@@ -133,7 +106,7 @@
public void setFirstLSN(long firstLSN) throws AsterixException {
// We make sure that this method is only called on an empty component so the first LSN is not set incorrectly
- firstLSNs[writeIndex] = firstLSN;
+ firstLSNs[writeIndex] = firstLSN;
}
public synchronized long getFirstLSN() {
@@ -142,10 +115,10 @@
return firstLSNs[readIndex];
}
- public synchronized boolean hasPendingFlush(){
+ public synchronized boolean hasPendingFlush() {
- for(int i=0; i<flushRequested.length; i++){
- if(flushRequested[i]){
+ for (int i = 0; i < flushRequested.length; i++) {
+ if (flushRequested[i]) {
return true;
}
}