Null merge branch 'gerrit/phoenix' into 'master'
As a number of commits from master have been replayed into
phoenix, perform a null merge commit to enable future merge
commits from phoenix to master
Change-Id: I00e9787583f396e4969904212f7b48e08d437e6b
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index a431f68..d2301a3 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -559,7 +559,7 @@
<id>asterix-gerrit-asterix-app</id>
<properties>
<test.excludes>
- **/CloudStorageTest.java,**/CloudStorageCancellationTest.java,**/CloudStorageGCSTest.java,**/CloudStorageUnstableTest.java,
+ **/CloudStorageTest.java,**/CloudStorageSparseTest,**/CloudStorageCancellationTest.java,**/CloudStorageGCSTest.java,**/CloudStorageUnstableTest.java,
**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,
**/SqlppExecutionColumnTest.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,
**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java,**/CloudPythonTest.java,
@@ -680,6 +680,7 @@
<properties>
<test.includes>
**/CloudStorageTest.java,
+ **/CloudStorageSparseTest.java,
**/CloudStorageCancellationTest.java,
**/SqlppSinglePointLookupExecutionTest.java, **/AwsS3*.java
</test.includes>
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index f5a496c..666702d 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -41,6 +41,7 @@
storage.memorycomponent.globalbudget=512MB
storage.io.scheduler=greedy
storage.filtered.memorycomponent.max.size=16MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index bb2e457..2f437a4 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -37,6 +37,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index cc83da0..c8903cb 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -37,6 +37,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageSparseTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageSparseTest.java
new file mode 100644
index 0000000..b7a264b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageSparseTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.cloud_storage;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.asterix.api.common.LocalCloudUtilAdobeMock;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.test.runtime.ExecutionTestUtil;
+import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.asterix.testframework.xml.Description;
+import org.apache.asterix.testframework.xml.TestCase;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+
+/**
+ * Run tests in cloud deployment environment
+ */
+@RunWith(Parameterized.class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class CloudStorageSparseTest {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final TestCaseContext tcCtx;
+ public static final String SUITE_TESTS = "testsuite_cloud_storage.xml";
+ public static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml";
+ public static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage-sparse.conf";
+ public static final String DELTA_RESULT_PATH = "results_cloud";
+ public static final String EXCLUDED_TESTS = "MP";
+
+ public static final String PLAYGROUND_CONTAINER = "playground";
+ public static final String MOCK_SERVER_REGION = "us-west-2";
+ public static final int MOCK_SERVER_PORT = 8001;
+ public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
+
+ public CloudStorageSparseTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH);
+ setupEnv(testExecutor);
+ }
+
+ public static void setupEnv(TestExecutor testExecutor) throws Exception {
+ LocalCloudUtilAdobeMock.startS3CloudEnvironment(true);
+ testExecutor.executorId = "cloud";
+ testExecutor.stripSubstring = "//DB:";
+ LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor);
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE_NAME);
+
+ // create the playground bucket and leave it empty, just for external collection-based tests
+ S3ClientBuilder builder = S3Client.builder();
+ URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+ .endpointOverride(endpoint);
+ S3Client client = builder.build();
+ client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
+ client.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ LocalCloudUtilAdobeMock.shutdownSilently();
+ }
+
+ @Parameters(name = "CloudStorageSparseTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
+ }
+
+ @Test
+ public void test() throws Exception {
+ List<TestCase.CompilationUnit> cu = tcCtx.getTestCase().getCompilationUnit();
+ Assume.assumeTrue(cu.size() > 1 || !EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription())));
+ LangExecutionUtil.test(tcCtx);
+ IBufferCache bufferCache;
+ for (NodeControllerService nc : ExecutionTestUtil.integrationUtil.ncs) {
+ bufferCache = ((INcApplicationContext) nc.getApplicationContext()).getBufferCache();
+ Assert.assertTrue(((BufferCache) bufferCache).isClean());
+ }
+ }
+
+ private static String getText(Description description) {
+ return description == null ? "" : description.getValue();
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppSinglePartitionExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppSinglePartitionExecutionTest.java
index d0823c7..6f19393 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppSinglePartitionExecutionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppSinglePartitionExecutionTest.java
@@ -28,7 +28,10 @@
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -68,6 +71,11 @@
@Test
public void test() throws Exception {
LangExecutionUtil.test(tcCtx);
+ IBufferCache bufferCache;
+ for (NodeControllerService nc : ExecutionTestUtil.integrationUtil.ncs) {
+ bufferCache = ((INcApplicationContext) nc.getApplicationContext()).getBufferCache();
+ Assert.assertTrue(((BufferCache) bufferCache).isClean());
+ }
}
private static void setNcEndpoints(TestExecutor testExecutor) {
diff --git a/asterixdb/asterix-app/src/test/resources/cc-analyze.conf b/asterixdb/asterix-app/src/test/resources/cc-analyze.conf
index c65bca8..7e5856a 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-analyze.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-analyze.conf
@@ -41,6 +41,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-azblob.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-azblob.conf
index 4af4fd0..409b99b 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-azblob.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-azblob.conf
@@ -41,6 +41,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
index 376252e..ee65f3c 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
@@ -41,6 +41,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf
index a9f2aac..8745d9d 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf
@@ -41,6 +41,7 @@
jvm.args=-Xmx4096m --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf
new file mode 100644
index 0000000..0842e01
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf
@@ -0,0 +1,77 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=target/tmp/asterix_nc1/iodevice1
+iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+credential.file=src/test/resources/security/passwd
+python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
+storage.page.zero.writer=sparse
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+credential.file=src/test/resources/security/passwd
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+compiler.ordered.fields=false
+compiler.internal.sanitycheck=true
+messaging.frame.size=4096
+messaging.frame.count=512
+cloud.deployment=true
+storage.buffercache.pagesize=32KB
+storage.partitioning=static
+cloud.storage.scheme=s3
+cloud.storage.bucket=cloud-storage-container
+cloud.storage.region=us-west-2
+cloud.storage.endpoint=http://127.0.0.1:8001
+cloud.storage.anonymous.auth=true
+cloud.storage.cache.policy=selective
+cloud.max.write.requests.per.second=2000
+cloud.max.read.requests.per.second=4000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
index 64fc40f..b3d1e2d 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
@@ -41,6 +41,8 @@
jvm.args=-Xmx4096m --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
+storage.page.zero.writer=default
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-columnar.conf b/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
index e01bbbd..9be2c16 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
@@ -40,6 +40,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
index aeb9361..23071d6 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
@@ -32,6 +32,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
@@ -52,4 +53,4 @@
compiler.ordered.fields=false
messaging.frame.size=4096
messaging.frame.count=512
-storage.buffercache.pagesize=32KB
+storage.buffercache.pagesize=32KB
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single.conf b/asterixdb/asterix-app/src/test/resources/cc-single.conf
index 76b29d7..6acd5b5 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single.conf
@@ -29,6 +29,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
index 086944b..735f3a3 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
@@ -45,6 +45,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index 075d865..bdcd537 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -41,6 +41,7 @@
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/flush/ASTERIXDB-3597/ASTERIXDB-3597.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/flush/ASTERIXDB-3597/ASTERIXDB-3597.003.query.sqlpp
index ff64a7e..8a7a3e9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/flush/ASTERIXDB-3597/ASTERIXDB-3597.003.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/flush/ASTERIXDB-3597/ASTERIXDB-3597.003.query.sqlpp
@@ -21,4 +21,5 @@
SELECT *
FROM ds
-WHERE id >= "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
\ No newline at end of file
+WHERE id >= "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ORDER BY id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/merge/ASTERIXDB-3597/ASTERIXDB-3597.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/merge/ASTERIXDB-3597/ASTERIXDB-3597.004.query.sqlpp
index ff64a7e..8a7a3e9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/merge/ASTERIXDB-3597/ASTERIXDB-3597.004.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/io/merge/ASTERIXDB-3597/ASTERIXDB-3597.004.query.sqlpp
@@ -21,4 +21,5 @@
SELECT *
FROM ds
-WHERE id >= "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
\ No newline at end of file
+WHERE id >= "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ORDER BY id;
\ No newline at end of file
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java
new file mode 100644
index 0000000..1ab3052
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.in;
+
+import java.io.EOFException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.SparseColumnPageZeroWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples.ColumnMultiPageZeroBufferProvider;
+
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+public final class MultiPageZeroByteBuffersReader {
+ private static final ByteBuffer EMPTY;
+ private final IntList notRequiredSegmentsIndexes;
+ private ColumnMultiPageZeroBufferProvider bufferProvider;
+ private final Int2IntMap segmentDir; // should I just create a buffer[numberOfSegments] instead?
+ private int maxBuffersSize;
+
+ static {
+ EMPTY = ByteBuffer.allocate(0);
+ EMPTY.limit(0);
+ }
+
+ private final List<ByteBuffer> buffers;
+
+ public MultiPageZeroByteBuffersReader() {
+ this.buffers = new ArrayList<>();
+ segmentDir = new Int2IntOpenHashMap();
+ notRequiredSegmentsIndexes = new IntArrayList();
+ segmentDir.defaultReturnValue(-1);
+ }
+
+ public void reset(IColumnBufferProvider pageZeroBufferProvider) throws HyracksDataException {
+ reset();
+ this.bufferProvider = (ColumnMultiPageZeroBufferProvider) pageZeroBufferProvider;
+ maxBuffersSize = bufferProvider.getNumberOfRemainingPages();
+ bufferProvider.readAll(buffers, segmentDir);
+ }
+
+ public void read(int segmentIndex, IPointable pointable, int position, int length)
+ throws EOFException, HyracksDataException {
+ if (segmentIndex < 0 || segmentIndex >= maxBuffersSize) {
+ throw new IndexOutOfBoundsException("Buffer index out of bounds: " + segmentIndex);
+ }
+
+ int bufferIndex = segmentDir.get(segmentIndex);
+ if (bufferIndex == -1) {
+ //Fill up the buffer
+ // this page was not pinned, because of the DefaultReadContext, as the pages were expected to be in disk.
+ // so read the required segment, and fill the buffer.
+ ByteBuffer buffer = bufferProvider.read(segmentIndex);
+ segmentDir.put(segmentIndex, buffers.size());
+ bufferIndex = buffers.size();
+ buffers.add(buffer);
+ }
+ ByteBuffer buffer = buffers.get(bufferIndex);
+ pointable.set(buffer.array(), position, length);
+ }
+
+ public int readOffset(long[] offsetColumnIndexPairs, int maxColumnsInZerothSegment, int numberOfColumnsInAPage,
+ int currentColumnIndex) {
+ int numberOfColumns = offsetColumnIndexPairs.length - 1;
+ for (Int2IntMap.Entry pair : segmentDir.int2IntEntrySet()) {
+ int segmentIndex = pair.getIntKey();
+ int bufferIndex = pair.getIntValue();
+ ByteBuffer buffer = buffers.get(bufferIndex);
+ int columnIndex = maxColumnsInZerothSegment + segmentIndex * numberOfColumnsInAPage;
+ int segmentOffset = 0;
+ for (int j = 0; j < numberOfColumnsInAPage; j++) {
+ int columnOffset = buffer.getInt(segmentOffset);
+ offsetColumnIndexPairs[currentColumnIndex] = IntPairUtil.of(columnOffset, columnIndex);
+ segmentOffset += DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ currentColumnIndex++;
+ columnIndex++;
+ if (columnIndex == numberOfColumns) {
+ break; // No need to read more columns from this buffer.
+ }
+ }
+ }
+ return currentColumnIndex;
+ }
+
+ public int readSparseOffset(long[] offsetColumnIndexPairs, int numberOfPageSegments, int numberOfColumnsInAPage,
+ int numberOfColumnsInLastSegment, int currentColumnIndex) {
+ for (Int2IntMap.Entry pair : segmentDir.int2IntEntrySet()) {
+ int segmentIndex = pair.getIntKey();
+ int bufferIndex = pair.getIntValue();
+ ByteBuffer buffer = buffers.get(bufferIndex);
+ int segmentOffset = 0;
+ int numberOfColumnsInSegment =
+ segmentIndex == numberOfPageSegments - 2 ? numberOfColumnsInLastSegment : numberOfColumnsInAPage;
+ for (int j = 0; j < numberOfColumnsInSegment; j++) {
+ int columnIndex = buffer.getInt(segmentOffset);
+ int columnOffset = buffer.getInt(segmentOffset + Integer.BYTES);
+ offsetColumnIndexPairs[currentColumnIndex++] = IntPairUtil.of(columnOffset, columnIndex);
+ segmentOffset += SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+ }
+ return currentColumnIndex;
+ }
+
+ public void readAllColumns(BitSet presentColumns, int numberOfPageSegments, int numberOfColumnsInAPage,
+ int numberOfColumnsInLastSegment) {
+ final int stride = SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ final int lastSegmentIndex = numberOfPageSegments - 2;
+
+ for (Int2IntMap.Entry entry : segmentDir.int2IntEntrySet()) {
+ final int segmentIndex = entry.getIntKey();
+ final int bufferIndex = entry.getIntValue();
+ final ByteBuffer buffer = buffers.get(bufferIndex);
+
+ final int columnsInSegment =
+ (segmentIndex == lastSegmentIndex) ? numberOfColumnsInLastSegment : numberOfColumnsInAPage;
+
+ int offset = 0;
+ int limit = columnsInSegment * stride;
+
+ while (offset < limit) {
+ presentColumns.set(buffer.getInt(offset));
+ offset += stride;
+ }
+ }
+ }
+
+ public void unPinNotRequiredSegments(BitSet pageZeroSegmentsPages, int numberOfPageZeroSegments)
+ throws HyracksDataException {
+ if (numberOfPageZeroSegments <= 1) {
+ // If there is only one segment, it is always pinned.
+ // So no need to unpin the segments.
+ return;
+ }
+ notRequiredSegmentsIndexes.clear();
+ // Start checking from index 1 (0th segment is always pinned)
+ int i = pageZeroSegmentsPages.nextClearBit(1);
+ while (i >= 1 && i < numberOfPageZeroSegments) {
+ int segmentIndex = i - 1; // Adjusted index for segmentDir
+
+ int bufferIndex = segmentDir.get(segmentIndex);
+ if (bufferIndex != -1) {
+ buffers.set(bufferIndex, EMPTY);
+ notRequiredSegmentsIndexes.add(bufferIndex);
+ segmentDir.remove(segmentIndex);
+ }
+
+ i = pageZeroSegmentsPages.nextClearBit(i + 1);
+ }
+ // Unpin the buffers that are not required anymore.
+ bufferProvider.releasePages(notRequiredSegmentsIndexes);
+ }
+
+ public int findColumnIndexInSegment(int segmentIndex, int columnIndex, int numberOfColumnsInSegment)
+ throws HyracksDataException {
+ if (segmentIndex < 0 || segmentIndex >= maxBuffersSize) {
+ throw new IndexOutOfBoundsException("Buffer index out of bounds: " + segmentIndex);
+ }
+ int bufferIndex = segmentDir.get(segmentIndex);
+ if (bufferIndex == -1) {
+ //Fill up the buffer
+ // this page was not pinned, because of the DefaultReadContext, as the pages were expected to be in disk.
+ // so read the required segment, and fill the buffer.
+ ByteBuffer buffer = bufferProvider.read(segmentIndex);
+ segmentDir.put(segmentIndex, buffers.size());
+ bufferIndex = buffers.size();
+ buffers.add(buffer);
+ }
+ ByteBuffer buffer = buffers.get(bufferIndex);
+ int start = 0;
+ int end = numberOfColumnsInSegment - 1;
+ while (start <= end) {
+ int mid = start + (end - start) / 2;
+ int midColumnIndex = buffer.getInt(mid * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE);
+ if (midColumnIndex == columnIndex) {
+ return mid; // found the column index
+ } else if (midColumnIndex < columnIndex) {
+ start = mid + 1;
+ } else {
+ end = mid - 1;
+ }
+ }
+
+ return -1;
+ }
+
+ public void reset() {
+ buffers.clear();
+ maxBuffersSize = 0;
+ segmentDir.clear();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractMultiBufferBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractMultiBufferBytesOutputStream.java
index 4b7c835..0c311f1 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractMultiBufferBytesOutputStream.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/AbstractMultiBufferBytesOutputStream.java
@@ -81,6 +81,34 @@
}
}
+ public void writeInSegment(int bufferIndex, int off, int val) throws IOException {
+ if (bufferIndex >= buffers.size()) {
+ int requiredBuffers = bufferIndex - buffers.size() + 1;
+ allocateBuffers(requiredBuffers);
+ }
+ ByteBuffer buffer = buffers.get(bufferIndex);
+ buffer.putInt(off, val);
+ }
+
+ public void writeInSegment(int bufferIndex, int off, int val1, int val2) throws IOException {
+ if (bufferIndex >= buffers.size()) {
+ int requiredBuffers = bufferIndex - buffers.size() + 1;
+ allocateBuffers(requiredBuffers);
+ }
+ ByteBuffer buffer = buffers.get(bufferIndex);
+ buffer.putInt(off, val1);
+ buffer.putInt(off + Integer.BYTES, val2);
+ }
+
+ public void writeInSegment(int bufferIndex, int off, long val) throws IOException {
+ if (bufferIndex >= buffers.size()) {
+ int requiredBuffers = bufferIndex - buffers.size() + 1;
+ allocateBuffers(requiredBuffers);
+ }
+ ByteBuffer buffer = buffers.get(bufferIndex);
+ buffer.putLong(off, val);
+ }
+
@Override
public void reserveByte(IReservedPointer pointer) throws IOException {
ensureCapacity(Byte.BYTES);
@@ -161,4 +189,10 @@
allocatedBytes += size;
return size;
}
+
+ protected void allocateBuffers(int count) throws HyracksDataException {
+ for (int i = 0; i < count; i++) {
+ allocateBuffer();
+ }
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ByteBufferOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ByteBufferOutputStream.java
index 8817ae6..5ac4f24 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ByteBufferOutputStream.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/ByteBufferOutputStream.java
@@ -35,6 +35,10 @@
return buffer.position() - startOffset;
}
+ public int getPosition() {
+ return buffer.position();
+ }
+
@Override
public void write(int b) throws IOException {
buffer.put((byte) b);
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
new file mode 100644
index 0000000..4ae0153
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.bytes.stream.out;
+
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+public final class MultiPersistentPageZeroBufferBytesOutputStream extends AbstractMultiBufferBytesOutputStream {
+ public MultiPersistentPageZeroBufferBytesOutputStream(Mutable<IColumnWriteMultiPageOp> multiPageOpRef) {
+ super(multiPageOpRef);
+ }
+
+ @Override
+ protected ByteBuffer confiscateNewBuffer() throws HyracksDataException {
+ return multiPageOpRef.getValue().confiscatePageZeroPersistent();
+ }
+
+ public void reset(int requiredPageSegments) throws HyracksDataException {
+ preReset();
+ allocateBuffers(requiredPageSegments); // these many buffers are required for page zero segments
+ }
+
+ @Override
+ protected void preReset() {
+ if (allocatedBytes > 0 || !buffers.isEmpty()) {
+ //This should not be the case, with the pageZero segments.
+ //As the stream should be finished after the flush.
+ throw new IllegalStateException("PageZero segments should already be finished after flush");
+ }
+ }
+
+ @Override
+ public void writeTo(OutputStream outputStream) {
+ throw new IllegalAccessError("Persistent stream cannot be written to other stream");
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java
index db1862c..218b2ba 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.column.filter;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -42,11 +41,12 @@
import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.IColumnValuesReaderFactory;
-import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
public class FilterAccessorProvider {
private final ObjectSchemaNode root;
@@ -138,16 +138,16 @@
return filterColumnReaders;
}
- public static void setFilterValues(List<IColumnRangeFilterValueAccessor> filterValueAccessors, ByteBuffer pageZero,
- int numberOfColumns) {
+ public static void setFilterValues(List<IColumnRangeFilterValueAccessor> filterValueAccessors,
+ ColumnBTreeReadLeafFrame frame) throws HyracksDataException {
+ IColumnPageZeroReader pageZeroReader = frame.getColumnPageZeroReader();
for (int i = 0; i < filterValueAccessors.size(); i++) {
ColumnRangeFilterValueAccessor accessor = (ColumnRangeFilterValueAccessor) filterValueAccessors.get(i);
int columnIndex = accessor.getColumnIndex();
long normalizedValue;
- if (columnIndex < numberOfColumns) {
- int filterOffset = pageZero.position() + columnIndex * AbstractColumnFilterWriter.FILTER_SIZE;
- normalizedValue =
- accessor.isMin() ? pageZero.getLong(filterOffset) : pageZero.getLong(filterOffset + Long.BYTES);
+ if (pageZeroReader.isValidColumn(columnIndex)) {
+ normalizedValue = accessor.isMin() ? pageZeroReader.getColumnFilterMin(columnIndex)
+ : pageZeroReader.getColumnFilterMax(columnIndex);
} else {
// Column is missing
normalizedValue = accessor.isMin() ? Long.MAX_VALUE : Long.MIN_VALUE;
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNestedNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNestedNode.java
index 187e460..31dcaae 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNestedNode.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNestedNode.java
@@ -19,6 +19,17 @@
package org.apache.asterix.column.metadata.schema;
public abstract class AbstractSchemaNestedNode extends AbstractSchemaNode {
+ //A nestedNode can initially be empty, which contributes
+ //to a missing column.
+ protected boolean missingInitiallyInBatch;
+
+ public void setMissingInitiallyInBatch(boolean missingInitiallyInBatch) {
+ this.missingInitiallyInBatch = missingInitiallyInBatch;
+ }
+
+ public boolean isMissingInitiallyInBatch() {
+ return missingInitiallyInBatch;
+ }
@Override
public final boolean isNested() {
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNode.java
index 622705c..4cfa731 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNode.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/AbstractSchemaNode.java
@@ -35,6 +35,16 @@
public abstract class AbstractSchemaNode {
private int counter;
+ //Indicates if all the columns of the children is to be included
+ private boolean needAllColumns;
+
+ //Needed for estimating the column count
+ protected int previousNumberOfColumns; // before transform
+ protected int numberOfColumns;
+ protected int numberOfVisitedColumnsInBatch;
+ private int newDiscoveredColumns;
+ private int visitedBatchVersion;
+ private int formerChildNullVersion;
public abstract ATypeTag getTypeTag();
@@ -48,6 +58,14 @@
counter++;
}
+ public void needAllColumns(boolean needAllColumns) {
+ this.needAllColumns = needAllColumns;
+ }
+
+ public boolean needAllColumns() {
+ return needAllColumns;
+ }
+
public final void setCounter(int counter) {
this.counter = counter;
}
@@ -90,4 +108,54 @@
throw new UnsupportedEncodingException(typeTag + " is not supported");
}
}
+
+ // Needed for estimating the number of columns.
+ public void setNumberOfVisitedColumnsInBatch(int numberOfVisitedColumnsInBatch) {
+ this.numberOfVisitedColumnsInBatch = numberOfVisitedColumnsInBatch;
+ }
+
+ public int getNumberOfVisitedColumnsInBatch() {
+ return numberOfVisitedColumnsInBatch;
+ }
+
+ public void setNewDiscoveredColumns(int newDiscoveredColumns) {
+ this.newDiscoveredColumns = newDiscoveredColumns;
+ }
+
+ public int getNewDiscoveredColumns() {
+ return newDiscoveredColumns;
+ }
+
+ public int getNumberOfColumns() {
+ return numberOfColumns;
+ }
+
+ public void incrementColumns(int deltaColumns) {
+ this.numberOfColumns += deltaColumns;
+ }
+
+ public int getDeltaColumnsChanged() {
+ if (previousNumberOfColumns != numberOfColumns) {
+ int diff = numberOfColumns - previousNumberOfColumns;
+ previousNumberOfColumns = numberOfColumns;
+ return diff;
+ }
+ return 0;
+ }
+
+ public void setFormerChildNull(int formerChildNullVersion) {
+ this.formerChildNullVersion = formerChildNullVersion;
+ }
+
+ public int formerChildNullVersion() {
+ return formerChildNullVersion;
+ }
+
+ public int getVisitedBatchVersion() {
+ return visitedBatchVersion;
+ }
+
+ public void setVisitedBatchVersion(int visitedBatchVersion) {
+ this.visitedBatchVersion = visitedBatchVersion;
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java
index 4cf2d23..497654c 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java
@@ -49,6 +49,7 @@
private final Int2IntMap fieldNameIndexToChildIndexMap;
private final List<AbstractSchemaNode> children;
private IntUnaryOperator nextIndex;
+ private boolean isEmptyObject = false;
public ObjectSchemaNode() {
fieldNameIndexToChildIndexMap = new Int2IntOpenHashMap();
@@ -72,7 +73,8 @@
}
children = new ArrayList<>();
- deserializeChildren(input, children, numberOfChildren, definitionLevels);
+ numberOfColumns = deserializeChildren(input, children, numberOfChildren, definitionLevels);
+ previousNumberOfColumns = numberOfColumns;
}
public AbstractSchemaNode getOrCreateChild(IValueReference fieldName, ATypeTag childTypeTag,
@@ -86,7 +88,13 @@
children.add(childIndex, newChild);
fieldNameIndexToChildIndexMap.put(fieldNameIndex, childIndex);
} else if (currentChild != newChild) {
+ numberOfColumns -= currentChild.getNumberOfColumns();
children.set(childIndex, newChild);
+ newChild.getDeltaColumnsChanged();
+ numberOfColumns += newChild.getNumberOfColumns();
+ } else {
+ // If the child is the same, we just update the number of columns
+ numberOfColumns += currentChild.getDeltaColumnsChanged();
}
return newChild;
@@ -98,13 +106,15 @@
children.add(child);
}
- public void setEmptyObject(FlushColumnMetadata columnMetadata) throws HyracksDataException {
+ public AbstractSchemaNode setEmptyObject(FlushColumnMetadata columnMetadata) throws HyracksDataException {
if (!children.isEmpty()) {
- return;
+ return null;
}
+ isEmptyObject = true;
AbstractSchemaNode emptyChild = columnMetadata.getOrCreateChild(null, ATypeTag.MISSING);
addChild(DUMMY_FIELD_NAME_INDEX, emptyChild);
nextIndex = this::emptyColumnIndex;
+ return emptyChild;
}
public AbstractSchemaNode getChild(int fieldNameIndex) {
@@ -187,11 +197,15 @@
}
}
- private static void deserializeChildren(DataInput input, List<AbstractSchemaNode> children, int numberOfChildren,
+ private static int deserializeChildren(DataInput input, List<AbstractSchemaNode> children, int numberOfChildren,
Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels) throws IOException {
+ int numberOfColumns = 0;
for (int i = 0; i < numberOfChildren; i++) {
- children.add(AbstractSchemaNode.deserialize(input, definitionLevels));
+ AbstractSchemaNode child = AbstractSchemaNode.deserialize(input, definitionLevels);
+ numberOfColumns += child.getNumberOfColumns();
+ children.add(child);
}
+ return numberOfColumns;
}
private int nextIndex(int fieldNameIndex) {
@@ -202,6 +216,11 @@
nextIndex = this::nextIndex;
fieldNameIndexToChildIndexMap.remove(DUMMY_FIELD_NAME_INDEX);
fieldNameIndexToChildIndexMap.put(fieldNameIndex, 0);
+ isEmptyObject = false;
return 0;
}
+
+ public boolean isEmptyObject() {
+ return isEmptyObject;
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
index 2503143..8124780 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
@@ -42,6 +42,11 @@
public UnionSchemaNode(AbstractSchemaNode child1, AbstractSchemaNode child2) {
children = new EnumMap<>(ATypeTag.class);
originalType = child1;
+ // this is a new node, update the number of columns
+ numberOfColumns = child1.getNumberOfColumns();
+ previousNumberOfColumns = numberOfColumns; // this is an older node
+ child2.getDeltaColumnsChanged();
+ numberOfColumns += child2.getNumberOfColumns();
putChild(child1);
putChild(child2);
}
@@ -54,10 +59,14 @@
ATypeTag originalTypeTag = ATypeTag.VALUE_TYPE_MAPPING[input.readByte()];
int numberOfChildren = input.readInt();
children = new EnumMap<>(ATypeTag.class);
+ int columnsCount = 0;
for (int i = 0; i < numberOfChildren; i++) {
AbstractSchemaNode child = AbstractSchemaNode.deserialize(input, definitionLevels);
+ columnsCount += child.getNumberOfColumns();
children.put(child.getTypeTag(), child);
}
+ numberOfColumns = columnsCount;
+ previousNumberOfColumns = numberOfColumns;
originalType = children.get(originalTypeTag);
}
@@ -65,6 +74,15 @@
children.put(child.getTypeTag(), child);
}
+ private void putChild(AbstractSchemaNode newChild, AbstractSchemaNode currentChild) {
+ if (currentChild != null && newChild.getTypeTag() == currentChild.getTypeTag()) {
+ numberOfColumns -= currentChild.getNumberOfColumns();
+ }
+ newChild.getDeltaColumnsChanged();
+ numberOfColumns += newChild.getNumberOfColumns();
+ children.put(newChild.getTypeTag(), newChild);
+ }
+
public AbstractSchemaNode getOriginalType() {
return originalType;
}
@@ -76,7 +94,9 @@
//The parent of a union child should be the actual parent
AbstractSchemaNode newChild = columnMetadata.getOrCreateChild(currentChild, normalizedTypeTag);
if (currentChild != newChild) {
- putChild(newChild);
+ putChild(newChild, currentChild);
+ } else {
+ numberOfColumns += newChild.getDeltaColumnsChanged();
}
return newChild;
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/AbstractCollectionSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/AbstractCollectionSchemaNode.java
index 8455864..1bcf1e9 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/AbstractCollectionSchemaNode.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/collection/AbstractCollectionSchemaNode.java
@@ -45,13 +45,24 @@
definitionLevels.put(this, new RunLengthIntArray());
}
item = AbstractSchemaNode.deserialize(input, definitionLevels);
+ numberOfColumns = item.getNumberOfColumns();
+ previousNumberOfColumns = numberOfColumns;
}
public final AbstractSchemaNode getOrCreateItem(ATypeTag childTypeTag, FlushColumnMetadata columnMetadata)
throws HyracksDataException {
AbstractSchemaNode newItem = columnMetadata.getOrCreateChild(item, childTypeTag);
- if (newItem != item) {
+ if (item == null) {
+ newItem.getDeltaColumnsChanged();
+ numberOfColumns += newItem.getNumberOfColumns();
item = newItem;
+ } else if (newItem != item) {
+ numberOfColumns -= item.getNumberOfColumns();
+ item = newItem;
+ newItem.getDeltaColumnsChanged();
+ numberOfColumns += newItem.getNumberOfColumns();
+ } else {
+ numberOfColumns += item.getDeltaColumnsChanged();
}
return item;
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/PrimitiveSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/PrimitiveSchemaNode.java
index 28d379d..af3d84b 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/PrimitiveSchemaNode.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/primitive/PrimitiveSchemaNode.java
@@ -37,12 +37,16 @@
this.columnIndex = columnIndex;
this.typeTag = typeTag;
this.primaryKey = primaryKey;
+ this.previousNumberOfColumns = 0;
+ this.numberOfColumns = 1;
}
public PrimitiveSchemaNode(ATypeTag typeTag, DataInput input) throws IOException {
this.typeTag = typeTag;
columnIndex = input.readInt();
primaryKey = input.readBoolean();
+ this.previousNumberOfColumns = 1;
+ this.numberOfColumns = 1;
}
public final int getColumnIndex() {
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaBuilderFromIATypeVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaBuilderFromIATypeVisitor.java
index c7d3df1..cbe4101 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaBuilderFromIATypeVisitor.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/SchemaBuilderFromIATypeVisitor.java
@@ -76,6 +76,7 @@
try {
AbstractSchemaNode itemNode = collectionNode.getOrCreateItem(itemType.getTypeTag(), columnMetadata);
itemType.accept(this, itemNode);
+ collectionNode.incrementColumns(itemNode.getDeltaColumnsChanged());
} catch (HyracksDataException e) {
throw new IllegalStateException(e);
}
@@ -154,5 +155,6 @@
AbstractSchemaNode child = objectNode.getOrCreateChild(fieldName, fieldType.getTypeTag(), columnMetadata);
fieldType.accept(this, child);
+ objectNode.incrementColumns(child.getDeltaColumnsChanged());
}
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java
index e3407a1..bc8f184 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.column.operation.lsm.flush;
+import java.util.BitSet;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
@@ -32,12 +33,16 @@
import org.apache.asterix.column.values.IColumnBatchWriter;
import org.apache.asterix.column.values.IColumnValuesWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
public final class BatchFinalizerVisitor implements ISchemaNodeVisitor<Void, AbstractSchemaNestedNode> {
private final FlushColumnMetadata columnSchemaMetadata;
private final IColumnValuesWriter[] primaryKeyWriters;
private final PriorityQueue<IColumnValuesWriter> orderedColumns;
+ private BitSet presentColumnsIndex;
+ private IColumnPageZeroWriter columnPageZeroWriter;
private int level;
+ private boolean needAllColumns;
public BatchFinalizerVisitor(FlushColumnMetadata columnSchemaMetadata) {
this.columnSchemaMetadata = columnSchemaMetadata;
@@ -50,15 +55,21 @@
level = -1;
}
- public int finalizeBatch(IColumnBatchWriter batchWriter, FlushColumnMetadata columnMetadata)
- throws HyracksDataException {
+ public void finalizeBatchColumns(FlushColumnMetadata columnMetadata, BitSet presentColumnsIndexes,
+ IColumnPageZeroWriter pageZeroWriter) throws HyracksDataException {
orderedColumns.clear();
+ this.presentColumnsIndex = presentColumnsIndexes;
+ this.needAllColumns = false;
+ this.columnPageZeroWriter = pageZeroWriter;
+ // is this needed to parse the whole schema??
columnMetadata.getRoot().accept(this, null);
if (columnMetadata.getMetaRoot() != null) {
columnMetadata.getMetaRoot().accept(this, null);
}
+ }
+ public int finalizeBatch(IColumnBatchWriter batchWriter) throws HyracksDataException {
batchWriter.writePrimaryKeyColumns(primaryKeyWriters);
return batchWriter.writeColumns(orderedColumns);
}
@@ -66,13 +77,18 @@
@Override
public Void visit(ObjectSchemaNode objectNode, AbstractSchemaNestedNode arg) throws HyracksDataException {
level++;
+ boolean previousNeedAllColumns = needAllColumns;
+ needAllColumns = needAllColumns | objectNode.needAllColumns();
columnSchemaMetadata.flushDefinitionLevels(level - 1, arg, objectNode);
List<AbstractSchemaNode> children = objectNode.getChildren();
for (int i = 0; i < children.size(); i++) {
children.get(i).accept(this, objectNode);
}
objectNode.setCounter(0);
+ objectNode.setNumberOfVisitedColumnsInBatch(0);
columnSchemaMetadata.clearDefinitionLevels(objectNode);
+ objectNode.needAllColumns(false);
+ needAllColumns = previousNeedAllColumns;
level--;
return null;
}
@@ -81,34 +97,50 @@
public Void visit(AbstractCollectionSchemaNode collectionNode, AbstractSchemaNestedNode arg)
throws HyracksDataException {
level++;
+ boolean previousNeedAllColumns = needAllColumns;
+ needAllColumns = needAllColumns | collectionNode.needAllColumns();
columnSchemaMetadata.flushDefinitionLevels(level - 1, arg, collectionNode);
collectionNode.getItemNode().accept(this, collectionNode);
collectionNode.setCounter(0);
+ collectionNode.setNumberOfVisitedColumnsInBatch(0);
columnSchemaMetadata.clearDefinitionLevels(collectionNode);
+ collectionNode.needAllColumns(false);
+ needAllColumns = previousNeedAllColumns;
level--;
return null;
}
@Override
public Void visit(UnionSchemaNode unionNode, AbstractSchemaNestedNode arg) throws HyracksDataException {
+ boolean previousNeedAllColumns = needAllColumns;
+ needAllColumns = needAllColumns | unionNode.needAllColumns();
columnSchemaMetadata.flushDefinitionLevels(level, arg, unionNode);
for (AbstractSchemaNode node : unionNode.getChildren().values()) {
node.accept(this, unionNode);
}
unionNode.setCounter(0);
+ unionNode.setNumberOfVisitedColumnsInBatch(0);
columnSchemaMetadata.clearDefinitionLevels(unionNode);
+ unionNode.needAllColumns(false);
+ needAllColumns = previousNeedAllColumns;
return null;
}
@Override
public Void visit(PrimitiveSchemaNode primitiveNode, AbstractSchemaNestedNode arg) throws HyracksDataException {
columnSchemaMetadata.flushDefinitionLevels(level, arg, primitiveNode);
- if (!primitiveNode.isPrimaryKey()) {
+ if (needAllColumns) {
+ presentColumnsIndex.set(primitiveNode.getColumnIndex());
+ }
+ // in case of DefaultWriter all non-primary columns should be included
+ if (!primitiveNode.isPrimaryKey() && columnPageZeroWriter.includeOrderedColumn(presentColumnsIndex,
+ primitiveNode.getColumnIndex(), needAllColumns)) {
orderedColumns.add(columnSchemaMetadata.getWriter(primitiveNode.getColumnIndex()));
}
//Prepare for the next batch
primitiveNode.setCounter(0);
+ primitiveNode.setNumberOfVisitedColumnsInBatch(0);
return null;
}
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java
index 71b561a..7b1a85b 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.column.operation.lsm.flush;
+import java.util.BitSet;
+
import org.apache.asterix.column.metadata.schema.AbstractSchemaNestedNode;
import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
@@ -41,6 +43,7 @@
private final FlushColumnMetadata columnMetadata;
private final VoidPointable nonTaggedValue;
private final ObjectSchemaNode root;
+ private final BitSet presentColumnsIndexes;
private AbstractSchemaNestedNode currentParent;
private int primaryKeysLength;
/**
@@ -49,12 +52,15 @@
* leaf node to avoid having a single column that spans to megabytes of pages.
*/
private int stringLengths;
+ private int currentBatchVersion;
- public ColumnTransformer(FlushColumnMetadata columnMetadata, ObjectSchemaNode root) {
+ public ColumnTransformer(FlushColumnMetadata columnMetadata, ObjectSchemaNode root, BitSet presentColumnsIndexes) {
this.columnMetadata = columnMetadata;
this.root = root;
+ this.presentColumnsIndexes = presentColumnsIndexes;
nonTaggedValue = new VoidPointable();
stringLengths = 0;
+ currentBatchVersion = 1;
}
public int getStringLengths() {
@@ -63,6 +69,7 @@
public void resetStringLengths() {
stringLengths = 0;
+ currentBatchVersion++;
}
/**
@@ -72,6 +79,7 @@
* @return the estimated size (possibly overestimated) of the primary key(s) columns
*/
public int transform(RecordLazyVisitablePointable pointable) throws HyracksDataException {
+ // clear the last present column indexes.
primaryKeysLength = 0;
pointable.accept(this, root);
return primaryKeysLength;
@@ -84,6 +92,8 @@
int start = tuple.getFieldStart(i);
ATypeTag tag = ATypeTag.VALUE_TYPE_MAPPING[bytes[start]];
nonTaggedValue.set(bytes, start + 1, tuple.getFieldLength(i) - 1);
+ // include the primary key column
+ presentColumnsIndexes.set(i);
IColumnValuesWriter writer = columnMetadata.getWriter(i);
writer.writeAntiMatter(tag, nonTaggedValue);
pkSize += writer.getEstimatedSize();
@@ -99,6 +109,13 @@
ObjectSchemaNode objectNode = (ObjectSchemaNode) arg;
currentParent = objectNode;
+
+ if (currentParent.getVisitedBatchVersion() != currentBatchVersion) {
+ currentParent.needAllColumns(false);
+ currentParent.setMissingInitiallyInBatch(false);
+ currentParent.setVisitedBatchVersion(currentBatchVersion);
+ }
+
for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
pointable.nextChild();
IValueReference fieldName = pointable.getFieldName();
@@ -113,6 +130,17 @@
if (pointable.getNumberOfChildren() == 0) {
// Set as empty object
objectNode.setEmptyObject(columnMetadata);
+ if (!objectNode.isMissingInitiallyInBatch() && objectNode.isEmptyObject()) {
+ objectNode.needAllColumns(true);
+ objectNode.setMissingInitiallyInBatch(true);
+ PrimitiveSchemaNode missingNode = (PrimitiveSchemaNode) objectNode.getChildren().get(0);
+ presentColumnsIndexes.set(missingNode.getColumnIndex());
+ }
+ } else {
+ if (objectNode.isMissingInitiallyInBatch()) {
+ objectNode.setMissingInitiallyInBatch(false);
+ objectNode.needAllColumns(false);
+ }
}
columnMetadata.exitNode(arg);
@@ -132,19 +160,34 @@
int missingLevel = columnMetadata.getLevel();
currentParent = collectionNode;
+ if (currentParent.getVisitedBatchVersion() != currentBatchVersion) {
+ currentParent.needAllColumns(false);
+ currentParent.setMissingInitiallyInBatch(false);
+ currentParent.setVisitedBatchVersion(currentBatchVersion);
+ }
+
+ // If it's an array, all column will be needed as anyone of them, will be a delegate.
+ currentParent.needAllColumns(true);
+
int numberOfChildren = pointable.getNumberOfChildren();
+ int newDiscoveredColumns = 0;
for (int i = 0; i < numberOfChildren; i++) {
pointable.nextChild();
ATypeTag childTypeTag = pointable.getChildTypeTag();
AbstractSchemaNode childNode = collectionNode.getOrCreateItem(childTypeTag, columnMetadata);
acceptActualNode(pointable.getChildVisitablePointable(), childNode);
+ currentParent.incrementColumns(childNode.getDeltaColumnsChanged());
+ newDiscoveredColumns += childNode.getNewDiscoveredColumns();
+ childNode.setNewDiscoveredColumns(0);
/*
* The array item may change (e.g., BIGINT --> UNION). Thus, new items would be considered as missing
*/
defLevels.add(missingLevel);
}
- // Add missing as a last element of the array to help indicate empty arrays
+ currentParent.setNewDiscoveredColumns(newDiscoveredColumns);
+ currentParent.setNumberOfVisitedColumnsInBatch(
+ currentParent.getNumberOfVisitedColumnsInBatch() + newDiscoveredColumns);
collectionNode.getOrCreateItem(ATypeTag.MISSING, columnMetadata);
defLevels.add(missingLevel);
@@ -159,6 +202,7 @@
columnMetadata.enterNode(currentParent, arg);
ATypeTag valueTypeTag = pointable.getTypeTag();
PrimitiveSchemaNode node = (PrimitiveSchemaNode) arg;
+ presentColumnsIndexes.set(node.getColumnIndex());
IColumnValuesWriter writer = columnMetadata.getWriter(node.getColumnIndex());
if (valueTypeTag == ATypeTag.MISSING) {
writer.writeLevel(columnMetadata.getLevel());
@@ -198,6 +242,7 @@
*/
AbstractSchemaNode actualNode = unionNode.getOriginalType();
acceptActualNode(pointable, actualNode);
+ currentParent.incrementColumns(actualNode.getDeltaColumnsChanged());
} else {
AbstractSchemaNode actualNode = unionNode.getOrCreateChild(pointable.getTypeTag(), columnMetadata);
pointable.accept(this, actualNode);
@@ -206,7 +251,8 @@
currentParent = previousParent;
columnMetadata.exitNode(node);
} else if (pointable.getTypeTag() == ATypeTag.NULL && node.isNested()) {
- columnMetadata.addNestedNull(currentParent, (AbstractSchemaNestedNode) node);
+ node.needAllColumns(true);
+ columnMetadata.addNestedNull(currentParent, (AbstractSchemaNestedNode) node, true);
} else {
pointable.accept(this, node);
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
index f987abe..b45d7d9 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
@@ -65,6 +65,7 @@
import org.apache.logging.log4j.Logger;
import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
/**
* Flush column metadata belongs to a flushing {@link ILSMMemoryComponent}
@@ -72,6 +73,7 @@
*/
public class FlushColumnMetadata extends AbstractColumnMetadata {
private static final Logger LOGGER = LogManager.getLogger();
+ public final IntOpenHashSet orderedColumns;
protected final Map<AbstractSchemaNestedNode, RunLengthIntArray> definitionLevels;
private final Mutable<IColumnWriteMultiPageOp> multiPageOpRef;
private final IFieldNamesDictionary fieldNamesDictionary;
@@ -93,6 +95,7 @@
super(datasetType, metaType, primaryKeys.size());
this.multiPageOpRef = multiPageOpRef;
this.columnWriterFactory = columnWriterFactory;
+ this.orderedColumns = new IntOpenHashSet();
definitionLevels = new HashMap<>();
columnWriters = new ArrayList<>();
level = -1;
@@ -133,6 +136,7 @@
super(datasetType, metaType, numPrimaryKeys);
this.multiPageOpRef = multiPageOpRef;
this.columnWriterFactory = columnWriterFactory;
+ this.orderedColumns = new IntOpenHashSet();
this.definitionLevels = definitionLevels;
this.columnWriters = columnWriters;
level = -1;
@@ -346,11 +350,18 @@
throws HyracksDataException {
AbstractSchemaNode currentChild = child;
ATypeTag normalizedTypeTag = getNormalizedTypeTag(childTypeTag);
+ boolean newChild = currentChild == null;
if (currentChild == null || normalizedTypeTag != ATypeTag.MISSING && normalizedTypeTag != ATypeTag.NULL
&& currentChild.getTypeTag() != ATypeTag.UNION
&& getNormalizedTypeTag(currentChild.getTypeTag()) != normalizedTypeTag) {
//Create a new child or union type if required type is different from the current child type
+ int visitedBatchVersion = newChild ? -1 : currentChild.getVisitedBatchVersion();
currentChild = createChild(child, childTypeTag);
+ //Missing will become UNION, hence only NULL and NULL will be replaced
+ if (!newChild && (child.getTypeTag() == ATypeTag.NULL)) {
+ //This will be a replaced child
+ currentChild.setFormerChildNull(visitedBatchVersion);
+ }
//Flag that the schema has changed
changed = true;
}
@@ -421,25 +432,30 @@
public void flushDefinitionLevels(int level, AbstractSchemaNestedNode parent, AbstractSchemaNode node)
throws HyracksDataException {
+ flushDefinitionLevels(level, parent, node, false);
+ }
+
+ public void addNestedNull(AbstractSchemaNestedNode parent, AbstractSchemaNestedNode node,
+ boolean includeChildColumns) throws HyracksDataException {
+ //Flush all definition levels from parent to the current node
+ flushDefinitionLevels(level, parent, node, includeChildColumns);
+ //Add null value (+2) to say that both the parent and the child are present
+ definitionLevels.get(node).add(ColumnValuesUtil.getNullMask(level + 2) | level);
+ node.incrementCounter();
+ }
+
+ private void flushDefinitionLevels(int level, AbstractSchemaNestedNode parent, AbstractSchemaNode node,
+ boolean includeChildColumns) throws HyracksDataException {
if (parent != null) {
RunLengthIntArray parentDefLevels = definitionLevels.get(parent);
if (node.getCounter() < parentDefLevels.getSize()) {
int parentMask = ColumnValuesUtil.getNullMask(level);
int childMask = ColumnValuesUtil.getNullMask(level + 1);
- flushDefinitionLevels(parentMask, childMask, parentDefLevels, node);
+ flushDefinitionLevels(parentMask, childMask, parentDefLevels, node, includeChildColumns);
}
}
}
- public void addNestedNull(AbstractSchemaNestedNode parent, AbstractSchemaNestedNode node)
- throws HyracksDataException {
- //Flush all definition levels from parent to the current node
- flushDefinitionLevels(level, parent, node);
- //Add null value (+2) to say that both the parent and the child are present
- definitionLevels.get(node).add(ColumnValuesUtil.getNullMask(level + 2) | level);
- node.incrementCounter();
- }
-
public void close() {
//Dereference multiPageOp
multiPageOpRef.setValue(null);
@@ -449,7 +465,7 @@
}
protected void flushDefinitionLevels(int parentMask, int childMask, RunLengthIntArray parentDefLevels,
- AbstractSchemaNode node) throws HyracksDataException {
+ AbstractSchemaNode node, boolean includeChildColumns) throws HyracksDataException {
int startIndex = node.getCounter();
if (node.isNested()) {
RunLengthIntArray childDefLevels = definitionLevels.get((AbstractSchemaNestedNode) node);
@@ -519,7 +535,7 @@
nullWriterIndexes.add(columnIndex);
createdChild = createChild(normalizedTypeTag);
int mask = ColumnValuesUtil.getNullMask(level);
- flushDefinitionLevels(mask, mask, defLevels, createdChild);
+ flushDefinitionLevels(mask, mask, defLevels, createdChild, false);
} else {
//Different type. Make union
createdChild = addDefinitionLevelsAndGet(new UnionSchemaNode(child, createChild(normalizedTypeTag)));
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWithMetaWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWithMetaWriter.java
index a1abf5e..f892643 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWithMetaWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWithMetaWriter.java
@@ -31,7 +31,8 @@
public FlushColumnTupleWithMetaWriter(FlushColumnMetadata columnMetadata, int pageSize, int maxNumberOfTuples,
double tolerance, int maxLeafNodeSize, IColumnWriteContext writeContext) {
super(columnMetadata, pageSize, maxNumberOfTuples, tolerance, maxLeafNodeSize, writeContext);
- metaColumnTransformer = new ColumnTransformer(columnMetadata, columnMetadata.getMetaRoot());
+ metaColumnTransformer =
+ new ColumnTransformer(columnMetadata, columnMetadata.getMetaRoot(), presentColumnsIndexes);
metaPointable = new TypedRecordLazyVisitablePointable(columnMetadata.getMetaType());
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
index 36dd6bc..bf76a6b 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
@@ -19,13 +19,17 @@
package org.apache.asterix.column.operation.lsm.flush;
import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.util.BitSet;
import org.apache.asterix.column.values.IColumnValuesWriter;
import org.apache.asterix.column.values.IColumnValuesWriterFactory;
import org.apache.asterix.column.values.writer.ColumnBatchWriter;
import org.apache.asterix.column.values.writer.ColumnValuesWriterFactory;
-import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.zero.PageZeroWriterFlavorSelector;
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.SparseColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter;
+import org.apache.asterix.column.zero.writers.multipage.SparseColumnMultiPageZeroWriter;
import org.apache.asterix.om.lazy.RecordLazyVisitablePointable;
import org.apache.asterix.om.lazy.TypedRecordLazyVisitablePointable;
import org.apache.commons.lang3.mutable.Mutable;
@@ -35,9 +39,14 @@
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnWriteContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriterFlavorSelector;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class FlushColumnTupleWriter extends AbstractColumnTupleWriter {
+ private static final Logger LOGGER = LogManager.getLogger();
protected final FlushColumnMetadata columnMetadata;
protected final NoWriteFlushColumnMetadata columnMetadataWithCurrentTuple;
@@ -50,13 +59,17 @@
private final int maxNumberOfTuples;
private final IColumnValuesWriter[] primaryKeyWriters;
private final int maxLeafNodeSize;
+ protected final BitSet presentColumnsIndexes;
protected int primaryKeysEstimatedSize;
+ protected final IColumnPageZeroWriterFlavorSelector pageZeroWriterFlavorSelector;
public FlushColumnTupleWriter(FlushColumnMetadata columnMetadata, int pageSize, int maxNumberOfTuples,
double tolerance, int maxLeafNodeSize, IColumnWriteContext writeContext) {
this.columnMetadata = columnMetadata;
- transformer = new ColumnTransformer(columnMetadata, columnMetadata.getRoot());
+ this.pageZeroWriterFlavorSelector = new PageZeroWriterFlavorSelector();
+ this.presentColumnsIndexes = new BitSet();
+ transformer = new ColumnTransformer(columnMetadata, columnMetadata.getRoot(), presentColumnsIndexes);
finalizer = new BatchFinalizerVisitor(columnMetadata);
writer = new ColumnBatchWriter(columnMetadata.getMultiPageOpRef(), pageSize, tolerance, writeContext);
this.maxNumberOfTuples = maxNumberOfTuples;
@@ -79,8 +92,8 @@
} catch (IOException e) {
throw new RuntimeException(e);
}
- transformerForCurrentTuple =
- new NoWriteColumnTransformer(columnMetadataWithCurrentTuple, columnMetadataWithCurrentTuple.getRoot());
+ transformerForCurrentTuple = new NoWriteColumnTransformer(columnMetadataWithCurrentTuple,
+ columnMetadataWithCurrentTuple.getRoot(), columnMetadataWithCurrentTuple.getMetaRoot());
}
@Override
@@ -89,7 +102,12 @@
}
@Override
- public final int getNumberOfColumns(boolean includeCurrentTupleColumns) {
+ public IColumnPageZeroWriterFlavorSelector getColumnPageZeroWriterFlavorSelector() {
+ return pageZeroWriterFlavorSelector;
+ }
+
+ @Override
+ public final int getAbsoluteNumberOfColumns(boolean includeCurrentTupleColumns) {
if (includeCurrentTupleColumns) {
return columnMetadataWithCurrentTuple.getNumberOfColumns();
} else {
@@ -109,10 +127,8 @@
}
@Override
- public final int getOccupiedSpace() {
- int numberOfColumns = getNumberOfColumns(true);
- int filterSize = numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE;
- return primaryKeysEstimatedSize + filterSize;
+ public final int getPrimaryKeysEstimatedSize() {
+ return primaryKeysEstimatedSize;
}
/**
@@ -129,6 +145,56 @@
}
@Override
+ public int getPageZeroWriterOccupiedSpace(int maxColumnsInPageZerothSegment, int bufferCapacity,
+ boolean includeCurrentTupleColumns, IColumnPageZeroWriter.ColumnPageZeroWriterType writerType) {
+ int spaceOccupiedByDefaultWriter;
+ int spaceOccupiedBySparseWriter;
+
+ if (writerType == IColumnPageZeroWriter.ColumnPageZeroWriterType.DEFAULT) {
+ // go for default multi-page writer
+ spaceOccupiedByDefaultWriter =
+ getSpaceOccupiedByDefaultWriter(maxColumnsInPageZerothSegment, includeCurrentTupleColumns);
+ return spaceOccupiedByDefaultWriter;
+ } else if (writerType == IColumnPageZeroWriter.ColumnPageZeroWriterType.SPARSE) {
+ // Maximum space occupied by the columns = maxColumnsInPageZerothSegment * (offset + filter size)
+ spaceOccupiedBySparseWriter = getSpaceOccupiedBySparseWriter(maxColumnsInPageZerothSegment, bufferCapacity);
+ return spaceOccupiedBySparseWriter;
+ }
+
+ spaceOccupiedByDefaultWriter =
+ getSpaceOccupiedByDefaultWriter(maxColumnsInPageZerothSegment, includeCurrentTupleColumns);
+ spaceOccupiedBySparseWriter = getSpaceOccupiedBySparseWriter(maxColumnsInPageZerothSegment, bufferCapacity);
+ pageZeroWriterFlavorSelector.switchPageZeroWriterIfNeeded(spaceOccupiedByDefaultWriter,
+ spaceOccupiedBySparseWriter);
+
+ return Math.min(spaceOccupiedBySparseWriter, spaceOccupiedByDefaultWriter);
+ }
+
+ private int getSpaceOccupiedByDefaultWriter(int maxColumnsInPageZerothSegment, boolean includeCurrentTupleColumns) {
+ int spaceOccupiedByDefaultWriter;
+ int totalNumberOfColumns = getAbsoluteNumberOfColumns(includeCurrentTupleColumns);
+ totalNumberOfColumns = Math.min(totalNumberOfColumns, maxColumnsInPageZerothSegment);
+ spaceOccupiedByDefaultWriter = DefaultColumnMultiPageZeroWriter.EXTENDED_HEADER_SIZE + totalNumberOfColumns
+ * (DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE + DefaultColumnPageZeroWriter.FILTER_SIZE);
+ return spaceOccupiedByDefaultWriter;
+ }
+
+ private int getSpaceOccupiedBySparseWriter(int maxColumnsInPageZerothSegment, int bufferCapacity) {
+ int presentColumns = transformerForCurrentTuple.getNumberOfVisitedColumnsInBatch();
+ int maximumNumberOfColumnsInASegment =
+ SparseColumnMultiPageZeroWriter.getMaximumNumberOfColumnsInAPage(bufferCapacity);
+ int numberOfExtraPagesRequired = presentColumns <= maxColumnsInPageZerothSegment ? 0
+ : (int) Math.ceil(
+ (double) (presentColumns - maxColumnsInPageZerothSegment) / maximumNumberOfColumnsInASegment);
+ int headerSpace = SparseColumnMultiPageZeroWriter.getHeaderSpace(numberOfExtraPagesRequired);
+ presentColumns = Math.min(presentColumns, maxColumnsInPageZerothSegment);
+
+ // space occupied by the sparse writer
+ return headerSpace + presentColumns
+ * (SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE + DefaultColumnPageZeroWriter.FILTER_SIZE);
+ }
+
+ @Override
public final void close() {
columnMetadata.close();
writer.close();
@@ -163,10 +229,37 @@
}
@Override
- public final int flush(ByteBuffer pageZero) throws HyracksDataException {
- writer.setPageZeroBuffer(pageZero, getNumberOfColumns(false), columnMetadata.getNumberOfPrimaryKeys());
+ public final int flush(IColumnPageZeroWriter pageZeroWriter) throws HyracksDataException {
+ int numberOfColumns = getAbsoluteNumberOfColumns(false);
+ finalizer.finalizeBatchColumns(columnMetadata, presentColumnsIndexes, pageZeroWriter);
+
+ //assertion logging
+ int presentColumnsCount = presentColumnsIndexes.cardinality();
+ int beforeTransformColumnCount = transformerForCurrentTuple.getBeforeTransformColumnsCount();
+ int currentTupleColumnsCount = transformerForCurrentTuple.getNumberOfVisitedColumnsInBatch();
+ if (beforeTransformColumnCount != presentColumnsCount || currentTupleColumnsCount != presentColumnsCount) {
+ LOGGER.debug("mismatch in column counts: beforeTransform={}, currentTuple={}, expected={}",
+ beforeTransformColumnCount, currentTupleColumnsCount, presentColumnsCount);
+ }
+
+ writer.setPageZeroWriter(pageZeroWriter, toIndexArray(presentColumnsIndexes), numberOfColumns);
+ return finalizer.finalizeBatch(writer);
+ }
+
+ @Override
+ public void reset() {
transformer.resetStringLengths();
- return finalizer.finalizeBatch(writer, columnMetadata);
+ transformerForCurrentTuple.reset();
+ presentColumnsIndexes.clear();
+ }
+
+ public static int[] toIndexArray(BitSet bitSet) {
+ int[] result = new int[bitSet.cardinality()];
+ int idx = 0;
+ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+ result[idx++] = i;
+ }
+ return result;
}
protected void writeRecord(ITupleReference tuple) throws HyracksDataException {
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java
index 5efab0e..89d812b 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java
@@ -23,6 +23,7 @@
import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
import org.apache.asterix.om.lazy.AbstractLazyVisitablePointable;
import org.apache.asterix.om.lazy.AbstractListLazyVisitablePointable;
import org.apache.asterix.om.lazy.FlatLazyVisitablePointable;
@@ -37,11 +38,17 @@
private final NoWriteFlushColumnMetadata columnMetadata;
private final ObjectSchemaNode root;
+ private final int metaColumnCount;
+ private int currentBatchVersion;
+ private int beforeTransformColumnsCount = 0;
private AbstractSchemaNestedNode currentParent;
- public NoWriteColumnTransformer(NoWriteFlushColumnMetadata columnMetadata, ObjectSchemaNode root) {
+ public NoWriteColumnTransformer(NoWriteFlushColumnMetadata columnMetadata, ObjectSchemaNode root,
+ ObjectSchemaNode metaRoot) {
this.columnMetadata = columnMetadata;
this.root = root;
+ this.metaColumnCount = metaRoot != null ? metaRoot.getNumberOfColumns() : 0;
+ currentBatchVersion = 1;
}
/**
@@ -51,10 +58,19 @@
* @return the estimated size (possibly overestimated) of the primary key(s) columns
*/
public int transform(RecordLazyVisitablePointable pointable) throws HyracksDataException {
+ beforeTransformColumnsCount = getNumberOfVisitedColumnsInBatch();
pointable.accept(this, root);
return 0;
}
+ public int getBeforeTransformColumnsCount() {
+ return beforeTransformColumnsCount;
+ }
+
+ public void reset() {
+ currentBatchVersion++;
+ }
+
@Override
public AbstractSchemaNode visit(RecordLazyVisitablePointable pointable, AbstractSchemaNode arg)
throws HyracksDataException {
@@ -62,6 +78,14 @@
ObjectSchemaNode objectNode = (ObjectSchemaNode) arg;
currentParent = objectNode;
+ int newDiscoveredColumns = 0;
+ if (currentParent.getVisitedBatchVersion() != currentBatchVersion) {
+ objectNode.setNumberOfVisitedColumnsInBatch(0);
+ objectNode.setVisitedBatchVersion(currentBatchVersion);
+ objectNode.setMissingInitiallyInBatch(false);
+ objectNode.needAllColumns(false);
+ }
+
for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
pointable.nextChild();
IValueReference fieldName = pointable.getFieldName();
@@ -69,12 +93,48 @@
if (childTypeTag != ATypeTag.MISSING) {
AbstractSchemaNode childNode = objectNode.getOrCreateChild(fieldName, childTypeTag, columnMetadata);
acceptActualNode(pointable.getChildVisitablePointable(), childNode);
+ int childDiscoveredColumns = childNode.getNewDiscoveredColumns();
+ if (childNode.formerChildNullVersion() == currentBatchVersion) {
+ //Missing or NULL contributed to one of the columns
+ childNode.setFormerChildNull(-1);
+ childDiscoveredColumns -= 1;
+ } else {
+ childNode.setFormerChildNull(-1);
+ }
+ newDiscoveredColumns += childDiscoveredColumns;
+ childNode.setNewDiscoveredColumns(0);
+ currentParent.incrementColumns(childNode.getDeltaColumnsChanged());
}
}
if (pointable.getNumberOfChildren() == 0) {
// Set as empty object
- objectNode.setEmptyObject(columnMetadata);
+ AbstractSchemaNode missingChild = objectNode.setEmptyObject(columnMetadata);
+ if (!objectNode.isMissingInitiallyInBatch() && objectNode.isEmptyObject()) {
+ objectNode.needAllColumns(true); // to include the missing column, while finalizing the batch.
+ objectNode.setMissingInitiallyInBatch(true);
+ if (missingChild != null) {
+ currentParent.incrementColumns(missingChild.getDeltaColumnsChanged());
+ }
+ newDiscoveredColumns += 1;
+ }
+ } else if (objectNode.isMissingInitiallyInBatch()) {
+ objectNode.setMissingInitiallyInBatch(false);
+ objectNode.needAllColumns(false);
+ }
+
+ if (objectNode.needAllColumns()) {
+ // parent is not array, but objectNode need all columns, because this node used to be null
+ int previousContribution = currentParent.getNumberOfVisitedColumnsInBatch();
+ newDiscoveredColumns = 0; // reset the new discovered columns
+ newDiscoveredColumns -= previousContribution;
+ newDiscoveredColumns += currentParent.getNumberOfColumns();
+ currentParent.setNewDiscoveredColumns(newDiscoveredColumns);
+ currentParent.setNumberOfVisitedColumnsInBatch(currentParent.getNumberOfColumns());
+ } else {
+ currentParent.setNewDiscoveredColumns(newDiscoveredColumns);
+ currentParent.setNumberOfVisitedColumnsInBatch(
+ currentParent.getNumberOfVisitedColumnsInBatch() + newDiscoveredColumns);
}
currentParent = previousParent;
@@ -89,16 +149,44 @@
AbstractCollectionSchemaNode collectionNode = (AbstractCollectionSchemaNode) arg;
currentParent = collectionNode;
+ if (currentParent.getVisitedBatchVersion() != currentBatchVersion) {
+ currentParent.setVisitedBatchVersion(currentBatchVersion);
+ currentParent.setNumberOfVisitedColumnsInBatch(0);
+ currentParent.setMissingInitiallyInBatch(false);
+ currentParent.needAllColumns(false);
+ }
+
+ currentParent.needAllColumns(true);
+
+ int newDiscoveredColumns = 0;
int numberOfChildren = pointable.getNumberOfChildren();
for (int i = 0; i < numberOfChildren; i++) {
pointable.nextChild();
ATypeTag childTypeTag = pointable.getChildTypeTag();
AbstractSchemaNode childNode = collectionNode.getOrCreateItem(childTypeTag, columnMetadata);
acceptActualNode(pointable.getChildVisitablePointable(), childNode);
+ currentParent.incrementColumns(childNode.getDeltaColumnsChanged());
+ int childDiscoveredColumns = childNode.getNewDiscoveredColumns();
+ if (childNode.formerChildNullVersion() == currentBatchVersion) {
+ //Missing or NULL contributed to one of the columns
+ childNode.setFormerChildNull(-1);
+ childDiscoveredColumns -= 1;
+ } else {
+ childNode.setFormerChildNull(-1);
+ }
+ newDiscoveredColumns += childDiscoveredColumns;
+ childNode.setNewDiscoveredColumns(0);
}
// Add missing as a last element of the array to help indicate empty arrays
collectionNode.getOrCreateItem(ATypeTag.MISSING, columnMetadata);
+
+ // need all the columns
+ newDiscoveredColumns = 0;
+ newDiscoveredColumns -= currentParent.getNumberOfVisitedColumnsInBatch();
+ newDiscoveredColumns += currentParent.getNumberOfColumns();
+ currentParent.setNewDiscoveredColumns(newDiscoveredColumns);
+ currentParent.setNumberOfVisitedColumnsInBatch(currentParent.getNumberOfColumns());
currentParent = previousParent;
return null;
}
@@ -106,6 +194,15 @@
@Override
public AbstractSchemaNode visit(FlatLazyVisitablePointable pointable, AbstractSchemaNode arg)
throws HyracksDataException {
+ PrimitiveSchemaNode node = (PrimitiveSchemaNode) arg;
+
+ if (node.getVisitedBatchVersion() != currentBatchVersion) {
+ //First time in this batch
+ node.setNumberOfVisitedColumnsInBatch(1);
+ node.setNewDiscoveredColumns(1);
+ node.setVisitedBatchVersion(currentBatchVersion);
+ }
+
return null;
}
@@ -119,23 +216,41 @@
ATypeTag childTypeTag = pointable.getTypeTag();
+ AbstractSchemaNode actualNode;
if (childTypeTag == ATypeTag.NULL || childTypeTag == ATypeTag.MISSING) {
/*
* NULL and MISSING are tracked since the start to be written in the originalType (i.e., the type
* before injecting a union between the parent and the original node).
*/
- AbstractSchemaNode actualNode = unionNode.getOriginalType();
+ actualNode = unionNode.getOriginalType();
acceptActualNode(pointable, actualNode);
} else {
- AbstractSchemaNode actualNode = unionNode.getOrCreateChild(pointable.getTypeTag(), columnMetadata);
+ actualNode = unionNode.getOrCreateChild(pointable.getTypeTag(), columnMetadata);
pointable.accept(this, actualNode);
}
-
+ unionNode.setNewDiscoveredColumns(actualNode.getNewDiscoveredColumns());
+ unionNode.setNumberOfVisitedColumnsInBatch(
+ unionNode.getNumberOfVisitedColumnsInBatch() + actualNode.getNewDiscoveredColumns());
+ actualNode.setNewDiscoveredColumns(0);
+ currentParent.incrementColumns(actualNode.getDeltaColumnsChanged());
currentParent = previousParent;
} else if (pointable.getTypeTag() == ATypeTag.NULL && node.isNested()) {
- columnMetadata.addNestedNull(currentParent, (AbstractSchemaNestedNode) node);
+ node.needAllColumns(true);
+ int previousContribution = node.getNumberOfVisitedColumnsInBatch();
+ int netContribution = node.getNumberOfColumns() - previousContribution;
+ node.setNewDiscoveredColumns(netContribution);
+ node.setNumberOfVisitedColumnsInBatch(node.getNumberOfColumns());
+ columnMetadata.addNestedNull(currentParent, (AbstractSchemaNestedNode) node, false);
} else {
pointable.accept(this, node);
}
}
-}
\ No newline at end of file
+
+ public int getNumberOfVisitedColumnsInBatch() {
+ //In case of batch of anti-matters, the current batch version is not equal to the root's visited batch version.
+ if (currentBatchVersion != root.getVisitedBatchVersion()) {
+ return columnMetadata.getNumberOfPrimaryKeys();
+ }
+ return root.getNumberOfVisitedColumnsInBatch() + metaColumnCount;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteFlushColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteFlushColumnMetadata.java
index 88e6cc2..7d77a97 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteFlushColumnMetadata.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteFlushColumnMetadata.java
@@ -111,7 +111,7 @@
@Override
protected void flushDefinitionLevels(int parentMask, int childMask, RunLengthIntArray parentDefLevels,
- AbstractSchemaNode node) throws HyracksDataException {
+ AbstractSchemaNode node, boolean includeChildColumns) throws HyracksDataException {
//NoOp
}
@@ -136,8 +136,8 @@
}
@Override
- public void addNestedNull(AbstractSchemaNestedNode parent, AbstractSchemaNestedNode node)
- throws HyracksDataException {
+ public void addNestedNull(AbstractSchemaNestedNode parent, AbstractSchemaNestedNode node,
+ boolean includeChildColumns) throws HyracksDataException {
//NoOp
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReader.java
index 4114f10..261221b 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleReader.java
@@ -20,6 +20,7 @@
import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
import org.apache.asterix.column.tuple.MergeColumnTupleReference;
+import org.apache.asterix.column.zero.PageZeroWriterFlavorSelector;
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
@@ -29,6 +30,7 @@
private final MergeColumnReadMetadata columnMetadata;
public MergeColumnTupleReader(AbstractColumnImmutableReadMetadata columnMetadata) {
+ super(new PageZeroWriterFlavorSelector());
this.columnMetadata = (MergeColumnReadMetadata) columnMetadata;
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java
index fc7b859..fb5cfdb 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java
@@ -18,17 +18,22 @@
*/
package org.apache.asterix.column.operation.lsm.merge;
-import java.nio.ByteBuffer;
+import java.util.BitSet;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnTupleWriter;
import org.apache.asterix.column.tuple.MergeColumnTupleReference;
import org.apache.asterix.column.util.RunLengthIntArray;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.IColumnValuesWriter;
import org.apache.asterix.column.values.writer.ColumnBatchWriter;
-import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
+import org.apache.asterix.column.zero.PageZeroWriterFlavorSelector;
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.SparseColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter;
+import org.apache.asterix.column.zero.writers.multipage.SparseColumnMultiPageZeroWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
@@ -36,6 +41,8 @@
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnWriteContext;
import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriterFlavorSelector;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -48,14 +55,19 @@
private final IColumnValuesWriter[] primaryKeyWriters;
private final PriorityQueue<IColumnValuesWriter> orderedColumns;
private final ColumnBatchWriter writer;
+ private final IColumnPageZeroWriterFlavorSelector pageZeroWriterFlavorSelector;
+ protected final BitSet presentColumnsIndexes;
private final int maxNumberOfTuples;
private int primaryKeysEstimatedSize;
private int numberOfAntiMatter;
+ private int numberOfTuples;
public MergeColumnTupleWriter(MergeColumnWriteMetadata columnMetadata, int pageSize, int maxNumberOfTuples,
double tolerance, int maxLeafNodeSize, IColumnWriteContext writeContext) {
this.columnMetadata = columnMetadata;
+ this.pageZeroWriterFlavorSelector = new PageZeroWriterFlavorSelector();
this.maxLeafNodeSize = maxLeafNodeSize;
+ this.presentColumnsIndexes = new BitSet();
List<IColumnTupleIterator> componentsTuplesList = columnMetadata.getComponentsTuples();
this.componentsTuples = new MergeColumnTupleReference[componentsTuplesList.size()];
int totalLength = 0;
@@ -64,6 +76,7 @@
MergeColumnTupleReference mergeTuple = (MergeColumnTupleReference) componentsTuplesList.get(i);
this.componentsTuples[i] = mergeTuple;
mergeTuple.registerEndOfPageCallBack(this::writeAllColumns);
+ mergeTuple.setColumnIndexes(presentColumnsIndexes);
totalNumberOfTuples += mergeTuple.getTupleCount();
totalLength += mergeTuple.getMergingLength();
}
@@ -95,7 +108,7 @@
}
@Override
- public int getNumberOfColumns(boolean includeCurrentTupleColumns) {
+ public int getAbsoluteNumberOfColumns(boolean includeCurrentTupleColumns) {
return columnMetadata.getNumberOfColumns();
}
@@ -105,15 +118,20 @@
}
@Override
- public int getOccupiedSpace() {
- int numberOfColumns = getNumberOfColumns(true);
- int filterSize = numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE;
- return primaryKeysEstimatedSize + filterSize;
+ public int getPrimaryKeysEstimatedSize() {
+ return primaryKeysEstimatedSize;
}
@Override
public void writeTuple(ITupleReference tuple) throws HyracksDataException {
MergeColumnTupleReference columnTuple = (MergeColumnTupleReference) tuple;
+ if (numberOfTuples == 0) {
+ // fill with the columnIndexes
+ for (MergeColumnTupleReference componentsTuple : componentsTuples) {
+ componentsTuple.fillColumnIndexes();
+ }
+ }
+ numberOfTuples++;
int componentIndex = columnTuple.getComponentIndex();
int skipCount = columnTuple.getAndResetSkipCount();
if (skipCount > 0) {
@@ -132,30 +150,113 @@
}
@Override
- public int flush(ByteBuffer pageZero) throws HyracksDataException {
+ public IColumnPageZeroWriterFlavorSelector getColumnPageZeroWriterFlavorSelector() {
+ return pageZeroWriterFlavorSelector;
+ }
+
+ @Override
+ public int getPageZeroWriterOccupiedSpace(int maxColumnsInPageZerothSegment, int bufferCapacity,
+ boolean includeCurrentTupleColumns, IColumnPageZeroWriter.ColumnPageZeroWriterType writerType) {
+ int spaceOccupiedByDefaultWriter;
+ int spaceOccupiedBySparseWriter;
+
+ if (writerType == IColumnPageZeroWriter.ColumnPageZeroWriterType.DEFAULT) {
+ // go for default multi-page writer
+ spaceOccupiedByDefaultWriter =
+ getSpaceOccupiedByDefaultWriter(maxColumnsInPageZerothSegment, includeCurrentTupleColumns);
+ return spaceOccupiedByDefaultWriter;
+ } else if (writerType == IColumnPageZeroWriter.ColumnPageZeroWriterType.SPARSE) {
+ // Maximum space occupied by the columns = maxColumnsInPageZerothSegment * (offset + filter size)
+ spaceOccupiedBySparseWriter = getSpaceOccupiedBySparseWriter(maxColumnsInPageZerothSegment, bufferCapacity);
+ return spaceOccupiedBySparseWriter;
+ }
+
+ spaceOccupiedBySparseWriter = getSpaceOccupiedBySparseWriter(maxColumnsInPageZerothSegment, bufferCapacity);
+ spaceOccupiedByDefaultWriter =
+ getSpaceOccupiedByDefaultWriter(maxColumnsInPageZerothSegment, includeCurrentTupleColumns);
+ pageZeroWriterFlavorSelector.switchPageZeroWriterIfNeeded(spaceOccupiedByDefaultWriter,
+ spaceOccupiedBySparseWriter);
+ return Math.min(spaceOccupiedBySparseWriter, spaceOccupiedByDefaultWriter);
+ }
+
+ private int getSpaceOccupiedByDefaultWriter(int maxColumnsInPageZerothSegment, boolean includeCurrentTupleColumns) {
+ int spaceOccupiedByDefaultWriter;
+ int totalNumberOfColumns = getAbsoluteNumberOfColumns(includeCurrentTupleColumns);
+ totalNumberOfColumns = Math.min(totalNumberOfColumns, maxColumnsInPageZerothSegment);
+ spaceOccupiedByDefaultWriter = DefaultColumnMultiPageZeroWriter.EXTENDED_HEADER_SIZE + totalNumberOfColumns
+ * (DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE + DefaultColumnPageZeroWriter.FILTER_SIZE);
+ return spaceOccupiedByDefaultWriter;
+ }
+
+ private int getSpaceOccupiedBySparseWriter(int maxColumnsInPageZerothSegment, int bufferCapacity) {
+ int presentColumns = presentColumnsIndexes.cardinality();
+ int maximumNumberOfColumnsInASegment =
+ SparseColumnMultiPageZeroWriter.getMaximumNumberOfColumnsInAPage(bufferCapacity);
+ int numberOfExtraPagesRequired = presentColumns <= maxColumnsInPageZerothSegment ? 0
+ : (int) Math.ceil(
+ (double) (presentColumns - maxColumnsInPageZerothSegment) / maximumNumberOfColumnsInASegment);
+ int headerSpace = SparseColumnMultiPageZeroWriter.getHeaderSpace(numberOfExtraPagesRequired);
+ presentColumns = Math.min(presentColumns, maxColumnsInPageZerothSegment);
+
+ // space occupied by the sparse writer
+ return headerSpace + presentColumns
+ * (SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE + DefaultColumnPageZeroWriter.FILTER_SIZE);
+ }
+
+ @Override
+ public int flush(IColumnPageZeroWriter pageZeroWriter) throws HyracksDataException {
+ // here the numberOfColumns is the total number of columns present in the LSM Index (across all disk components)
+ // Hence, a merge will fail if union(NumberOfColumns(D1) + NumberOfColumns(D2) + ... + NumberOfColumns(DN)) >
+ // pageZero space, and since the merged page contains this many number of columns, the first flush will fail.
int numberOfColumns = columnMetadata.getNumberOfColumns();
int numberOfPrimaryKeys = columnMetadata.getNumberOfPrimaryKeys();
+
+ // If writtenComponents is not empty, process non-key columns
if (writtenComponents.getSize() > 0) {
writeNonKeyColumns();
writtenComponents.reset();
}
- for (int i = numberOfPrimaryKeys; i < numberOfColumns; i++) {
- orderedColumns.add(columnMetadata.getWriter(i));
+
+ // Iterate over the BitSet (presentColumnsIndexes) to get the indexes of the set bits
+ for (int columnIndex = presentColumnsIndexes.nextSetBit(0); columnIndex >= 0; columnIndex =
+ presentColumnsIndexes.nextSetBit(columnIndex + 1)) {
+ if (columnIndex < numberOfPrimaryKeys) {
+ continue; // Skip primary key columns
+ }
+ orderedColumns.add(columnMetadata.getWriter(columnIndex));
}
- writer.setPageZeroBuffer(pageZero, numberOfColumns, numberOfPrimaryKeys);
+
+ // Reset pageZeroWriter based on the writer
+ writer.setPageZeroWriter(pageZeroWriter, toIndexArray(presentColumnsIndexes), numberOfColumns);
+
+ // Write primary key columns
writer.writePrimaryKeyColumns(primaryKeyWriters);
+
+ // Write the other columns and get the total length
int totalLength = writer.writeColumns(orderedColumns);
+ // Reset numberOfAntiMatter (assuming this is part of the logic)
numberOfAntiMatter = 0;
+
return totalLength;
}
+ public static int[] toIndexArray(BitSet bitSet) {
+ return FlushColumnTupleWriter.toIndexArray(bitSet);
+ }
+
@Override
public void close() {
columnMetadata.close();
writer.close();
}
+ @Override
+ public void reset() {
+ presentColumnsIndexes.clear();
+ numberOfTuples = 0;
+ }
+
private void writePrimaryKeys(MergeColumnTupleReference columnTuple) throws HyracksDataException {
int primaryKeySize = 0;
for (int i = 0; i < columnMetadata.getNumberOfPrimaryKeys(); i++) {
@@ -171,16 +272,22 @@
for (int i = 0; i < writtenComponents.getNumberOfBlocks(); i++) {
int componentIndex = writtenComponents.getBlockValue(i);
if (componentIndex < 0) {
- //Skip writing values of deleted tuples
+ // Skip writing values of deleted tuples
componentIndex = clearAntimatterIndicator(componentIndex);
skipReaders(componentIndex, writtenComponents.getBlockSize(i));
continue;
}
MergeColumnTupleReference componentTuple = componentsTuples[componentIndex];
int count = writtenComponents.getBlockSize(i);
- for (int j = columnMetadata.getNumberOfPrimaryKeys(); j < columnMetadata.getNumberOfColumns(); j++) {
- IColumnValuesReader columnReader = componentTuple.getReader(j);
- IColumnValuesWriter columnWriter = columnMetadata.getWriter(j);
+
+ // Iterate over the set bits in presentColumnsIndexes
+ for (int columnIndex = presentColumnsIndexes.nextSetBit(0); columnIndex >= 0; columnIndex =
+ presentColumnsIndexes.nextSetBit(columnIndex + 1)) {
+ if (columnIndex < columnMetadata.getNumberOfPrimaryKeys()) {
+ continue;
+ }
+ IColumnValuesReader columnReader = componentTuple.getReader(columnIndex);
+ IColumnValuesWriter columnWriter = columnMetadata.getWriter(columnIndex);
writeColumn(i, componentIndex, columnReader, columnWriter, count);
}
}
@@ -201,8 +308,13 @@
private void skipReaders(int componentIndex, int count) throws HyracksDataException {
MergeColumnTupleReference componentTuple = componentsTuples[componentIndex];
try {
- for (int j = columnMetadata.getNumberOfPrimaryKeys(); j < columnMetadata.getNumberOfColumns(); j++) {
- IColumnValuesReader columnReader = componentTuple.getReader(j);
+ // Iterate over the set bits in presentColumnsIndexes
+ for (int columnIndex = presentColumnsIndexes.nextSetBit(0); columnIndex >= 0; columnIndex =
+ presentColumnsIndexes.nextSetBit(columnIndex + 1)) {
+ if (columnIndex < columnMetadata.getNumberOfPrimaryKeys()) {
+ continue;
+ }
+ IColumnValuesReader columnReader = componentTuple.getReader(columnIndex);
columnReader.skip(count);
}
} catch (ColumnarValueException e) {
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleReader.java
index 36e47ec..6afa274 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleReader.java
@@ -21,6 +21,7 @@
import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
import org.apache.asterix.column.tuple.QueryColumnTupleReference;
import org.apache.asterix.column.tuple.QueryColumnWithMetaTupleReference;
+import org.apache.asterix.column.zero.PageZeroWriterFlavorSelector;
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
@@ -30,6 +31,7 @@
private final QueryColumnMetadata columnMetadata;
public QueryColumnTupleReader(AbstractColumnImmutableReadMetadata columnMetadata) {
+ super(new PageZeroWriterFlavorSelector());
this.columnMetadata = (QueryColumnMetadata) columnMetadata;
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
index 6e13113..eb12a27 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
@@ -18,14 +18,13 @@
*/
package org.apache.asterix.column.tuple;
-import java.nio.ByteBuffer;
+import java.util.BitSet;
import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream;
import org.apache.asterix.column.operation.lsm.merge.IEndOfPageCallBack;
import org.apache.asterix.column.operation.lsm.merge.MergeColumnReadMetadata;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
-import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
@@ -38,6 +37,7 @@
private final IColumnValuesReader[] columnReaders;
private int skipCount;
private IEndOfPageCallBack endOfPageCallBack;
+ private BitSet presentColumnIndexes;
private int mergingLength;
public MergeColumnTupleReference(int componentIndex, ColumnBTreeReadLeafFrame frame,
@@ -65,9 +65,9 @@
}
@Override
- protected boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples) {
+ protected boolean startNewPage(int numberOfTuples) {
//Skip filters
- pageZero.position(pageZero.position() + numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE);
+ frame.skipFilters();
// skip count is always start from zero as no "search" is conducted during a merge
this.skipCount = 0;
mergingLength = 0;
@@ -90,6 +90,15 @@
}
@Override
+ public void newPage() throws HyracksDataException {
+ super.newPage();
+ // the tuples are being read, meanwhile MegaLeaf changed
+ if (presentColumnIndexes != null) {
+ frame.getAllColumns(presentColumnIndexes);
+ }
+ }
+
+ @Override
protected void startColumnFilter(IColumnBufferProvider buffersProvider, int ordinal, int numberOfTuples)
throws HyracksDataException {
// NoOp
@@ -145,4 +154,12 @@
}
};
}
+
+ public void setColumnIndexes(BitSet presentColumnsIndexes) {
+ this.presentColumnIndexes = presentColumnsIndexes;
+ }
+
+ public void fillColumnIndexes() {
+ frame.getAllColumns(presentColumnIndexes);
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
index 8113288..09539b9 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.column.tuple;
-import java.nio.ByteBuffer;
import java.util.List;
import org.apache.asterix.column.assembler.value.MissingValueGetter;
@@ -36,7 +35,6 @@
import org.apache.asterix.column.operation.query.QueryColumnMetadata;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
-import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
@@ -93,14 +91,13 @@
}
@Override
- protected boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples)
- throws HyracksDataException {
+ protected boolean startNewPage(int numberOfTuples) throws HyracksDataException {
//Skip to filters
- pageZero.position(pageZero.position() + numberOfColumns * Integer.BYTES);
+ frame.skipColumnOffsets();
//Set filters' values
- FilterAccessorProvider.setFilterValues(filterValueAccessors, pageZero, numberOfColumns);
+ FilterAccessorProvider.setFilterValues(filterValueAccessors, frame);
//Skip filters
- pageZero.position(pageZero.position() + numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE);
+ frame.skipFilters();
//Check if we should read all column pages
boolean readColumns = rangeFilterEvaluator.evaluate();
assembler.reset(readColumns ? numberOfTuples : 0);
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
index 2b8da6d..f3a4fcf 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.column.tuple;
-import java.nio.ByteBuffer;
import java.util.List;
import org.apache.asterix.column.assembler.value.MissingValueGetter;
@@ -37,7 +36,6 @@
import org.apache.asterix.column.operation.query.QueryColumnWithMetaMetadata;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
-import org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
@@ -97,14 +95,13 @@
}
@Override
- protected boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples)
- throws HyracksDataException {
+ protected boolean startNewPage(int numberOfTuples) throws HyracksDataException {
//Skip to filters
- pageZero.position(pageZero.position() + numberOfColumns * Integer.BYTES);
+ frame.skipColumnOffsets();
//Set filters' values
- FilterAccessorProvider.setFilterValues(filterValueAccessors, pageZero, numberOfColumns);
+ FilterAccessorProvider.setFilterValues(filterValueAccessors, frame);
//Skip filters
- pageZero.position(pageZero.position() + numberOfColumns * AbstractColumnFilterWriter.FILTER_SIZE);
+ frame.skipFilters();
//Check if we should read all column pages
boolean readColumns = rangeFilterEvaluator.evaluate();
assembler.reset(readColumns ? numberOfTuples : 0);
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnBatchWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnBatchWriter.java
index 063743d..ade1d38 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnBatchWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnBatchWriter.java
@@ -18,13 +18,34 @@
*/
package org.apache.asterix.column.values;
-import java.nio.ByteBuffer;
import java.util.PriorityQueue;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+/**
+ * Interface for writing column batch data to storage pages.
+ *
+ * This interface abstracts the process of writing columnar data, supporting both
+ * dense and sparse column layouts through the use of pluggable page zero writers.
+ * The writer handles page zero metadata, primary key storage, and column data placement
+ * across multiple pages with optimal space utilization.
+ */
public interface IColumnBatchWriter {
- void setPageZeroBuffer(ByteBuffer pageZeroBuffer, int numberOfColumns, int numberOfPrimaryKeys);
+
+ /**
+ * Configures the page zero writer for this batch.
+ *
+ * This method replaces the previous direct buffer approach with a more flexible
+ * abstraction that supports different page zero layouts (default vs sparse).
+ * The writer will be used to manage column offsets, filters, and primary key storage.
+ *
+ * @param pageZeroWriter The writer implementation for page zero operations
+ * @param presentColumnsIndexes Array of column indexes that contain data in this batch
+ * @param numberOfColumns Total number of columns in the schema
+ */
+ void setPageZeroWriter(IColumnPageZeroWriter pageZeroWriter, int[] presentColumnsIndexes, int numberOfColumns)
+ throws HyracksDataException;
/**
* Writes the primary keys' values to Page0
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriter.java
index 2e2aa9e..cb14d51 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/IColumnValuesWriter.java
@@ -20,17 +20,17 @@
import java.io.DataOutput;
import java.io.IOException;
-import java.io.OutputStream;
import org.apache.asterix.column.util.RunLengthIntArray;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IValuesWriter;
/**
* Column writer for values
*/
-public interface IColumnValuesWriter {
+public interface IColumnValuesWriter extends IValuesWriter {
/**
* Reset the writer
@@ -124,13 +124,6 @@
long getNormalizedMaxValue();
/**
- * Flush the columns value to output stream
- *
- * @param out output stream
- */
- void flush(OutputStream out) throws HyracksDataException;
-
- /**
* Close the writer and release all allocated buffers
*/
void close();
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnBatchWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnBatchWriter.java
index ee12140..e1bf8e3 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnBatchWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/ColumnBatchWriter.java
@@ -18,12 +18,8 @@
*/
package org.apache.asterix.column.values.writer;
-import static org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter.FILTER_SIZE;
-
-import java.nio.ByteBuffer;
import java.util.PriorityQueue;
-import org.apache.asterix.column.bytes.stream.out.ByteBufferOutputStream;
import org.apache.asterix.column.bytes.stream.out.MultiPersistentBufferBytesOutputStream;
import org.apache.asterix.column.bytes.stream.out.pointer.IReservedPointer;
import org.apache.asterix.column.values.IColumnBatchWriter;
@@ -32,21 +28,22 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnWriteContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
/**
- * A writer for a batch columns' values
+ * A writer for a batch columns' values.
+ * This implementation abstracts the page zero operations using IColumnPageZeroWriter,
+ * which allows for supporting different column formats including sparse columns.
*/
public final class ColumnBatchWriter implements IColumnBatchWriter {
- private final ByteBufferOutputStream primaryKeys;
private final MultiPersistentBufferBytesOutputStream columns;
private final int pageSize;
private final double tolerance;
private final IColumnWriteContext writeContext;
private final IReservedPointer columnLengthPointer;
- private ByteBuffer pageZero;
- private int columnsOffset;
- private int filtersOffset;
- private int primaryKeysOffset;
+ // The writer for page zero, which handles all page zero operations including
+ // column offsets and filters. This abstraction supports both default and sparse column formats.
+ private IColumnPageZeroWriter pageZeroWriter;
private int nonKeyColumnStartOffset;
public ColumnBatchWriter(Mutable<IColumnWriteMultiPageOp> multiPageOpRef, int pageSize, double tolerance,
@@ -54,35 +51,31 @@
this.pageSize = pageSize;
this.tolerance = tolerance;
this.writeContext = writeContext;
- primaryKeys = new ByteBufferOutputStream();
columns = new MultiPersistentBufferBytesOutputStream(multiPageOpRef);
columnLengthPointer = columns.createPointer();
}
- @Override
- public void setPageZeroBuffer(ByteBuffer pageZero, int numberOfColumns, int numberOfPrimaryKeys) {
- this.pageZero = pageZero;
- int offset = pageZero.position();
-
- columnsOffset = offset;
- offset += numberOfColumns * Integer.BYTES;
-
- filtersOffset = offset;
- offset += numberOfColumns * FILTER_SIZE;
-
- pageZero.position(offset);
- primaryKeysOffset = offset;
- primaryKeys.reset(pageZero);
- nonKeyColumnStartOffset = pageZero.capacity();
+ /**
+ * Sets the page zero writer implementation and initializes it.
+ * This method replaces the direct page zero buffer manipulation with a more abstracted approach,
+ * which allows for different page zero layouts (default or sparse).
+ *
+ * @param pageZeroWriter The writer implementation for page zero operations
+ * @param presentColumnsIndexes Array containing the indexes of columns present in this batch
+ * @param numberOfColumns Total number of columns in the schema
+ */
+ public void setPageZeroWriter(IColumnPageZeroWriter pageZeroWriter, int[] presentColumnsIndexes,
+ int numberOfColumns) throws HyracksDataException {
+ this.pageZeroWriter = pageZeroWriter;
+ pageZeroWriter.resetBasedOnColumns(presentColumnsIndexes, numberOfColumns);
+ pageZeroWriter.allocateColumns();
+ nonKeyColumnStartOffset = pageZeroWriter.getPageZeroBufferCapacity();
}
@Override
public void writePrimaryKeyColumns(IColumnValuesWriter[] primaryKeyWriters) throws HyracksDataException {
- for (int i = 0; i < primaryKeyWriters.length; i++) {
- IColumnValuesWriter writer = primaryKeyWriters[i];
- setColumnOffset(i, primaryKeysOffset + primaryKeys.size());
- writer.flush(primaryKeys);
- }
+ // Delegate primary key column writing to the page zero writer
+ pageZeroWriter.writePrimaryKeyColumns(primaryKeyWriters);
}
@Override
@@ -102,6 +95,14 @@
writeContext.close();
}
+ /**
+ * Writes a column's data to the batch.
+ * This method handles column data placement, ensuring optimal space usage and minimizing page splits.
+ * It also records column offsets and filter values in page zero through the pageZeroWriter.
+ *
+ * @param writer The column values writer containing data to be written
+ * @throws HyracksDataException If an error occurs during writing
+ */
private void writeColumn(IColumnValuesWriter writer) throws HyracksDataException {
boolean overlapping = true;
if (!hasEnoughSpace(columns.getCurrentBufferPosition(), writer)) {
@@ -118,9 +119,21 @@
writeContext.startWritingColumn(columnIndex, overlapping);
int columnRelativeOffset = columns.size();
columns.reserveInteger(columnLengthPointer);
- setColumnOffset(writer.getColumnIndex(), nonKeyColumnStartOffset + columnRelativeOffset);
- writeFilter(writer);
+ // Get the relative column index within the current page zero layout
+ // This mapping is particularly important for sparse columns where not all columns may be present
+ int relativeColumnIndex = pageZeroWriter.getRelativeColumnIndex(columnIndex);
+
+ // Record the column's absolute offset in page zero using the writer abstraction
+ pageZeroWriter.putColumnOffset(columnIndex, relativeColumnIndex,
+ nonKeyColumnStartOffset + columnRelativeOffset);
+
+ // Store column filter information (min/max values) in page zero
+ // This allows for faster filtering during query execution
+ pageZeroWriter.putColumnFilter(relativeColumnIndex, writer.getNormalizedMinValue(),
+ writer.getNormalizedMaxValue());
+
+ // Write the actual column data
writer.flush(columns);
int length = columns.size() - columnRelativeOffset;
@@ -128,6 +141,15 @@
writeContext.endWritingColumn(columnIndex, length);
}
+ /**
+ * Determines if there is enough space in the current buffer for a column.
+ * This method implements a space management strategy that balances between
+ * optimal buffer utilization and minimizing column splits across pages.
+ *
+ * @param bufferPosition Current position in the buffer
+ * @param columnWriter The column writer with data to be written
+ * @return true if there is enough space, false otherwise
+ */
private boolean hasEnoughSpace(int bufferPosition, IColumnValuesWriter columnWriter) {
if (bufferPosition == 0) {
// if the current buffer is empty, then use it
@@ -159,14 +181,4 @@
*/
return freeSpace > columnSize || remainingPercentage >= tolerance;
}
-
- private void setColumnOffset(int columnIndex, int offset) {
- pageZero.putInt(columnsOffset + Integer.BYTES * columnIndex, offset);
- }
-
- private void writeFilter(IColumnValuesWriter writer) {
- int offset = filtersOffset + writer.getColumnIndex() * FILTER_SIZE;
- pageZero.putLong(offset, writer.getNormalizedMinValue());
- pageZero.putLong(offset + Long.BYTES, writer.getNormalizedMaxValue());
- }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/AbstractColumnFilterWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/AbstractColumnFilterWriter.java
index abbe314..458e24c 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/AbstractColumnFilterWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/filters/AbstractColumnFilterWriter.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.data.std.api.IValueReference;
public abstract class AbstractColumnFilterWriter {
- public static final int FILTER_SIZE = Long.BYTES * 2;
public void addLong(long value) {
throw new UnsupportedOperationException(getClass().getName());
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
new file mode 100644
index 0000000..5deead4
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero;
+
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter.DEFAULT_WRITER_FLAG;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter.MULTI_PAGE_DEFAULT_WRITER_FLAG;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter.MULTI_PAGE_SPARSE_WRITER_FLAG;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter.SPARSE_WRITER_FLAG;
+
+import org.apache.asterix.column.zero.readers.DefaultColumnPageZeroReader;
+import org.apache.asterix.column.zero.readers.SparseColumnPageZeroReader;
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.SparseColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroReader;
+import org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter;
+import org.apache.asterix.column.zero.writers.multipage.SparseColumnMultiPageZeroReader;
+import org.apache.asterix.column.zero.writers.multipage.SparseColumnMultiPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriterFlavorSelector;
+
+import it.unimi.dsi.fastutil.bytes.Byte2ObjectArrayMap;
+
+/**
+ * Selector that chooses between different page zero writer implementations based on space efficiency.
+ *<p>
+ * This class implements an optimization strategy for sparse columns:
+ * - Default writer: Allocates space for all columns in the schema (suitable for dense data)
+ * - Sparse writer: Only allocates space for present columns (suitable for sparse data)
+ * - Multi-page writers: Variants of the above writers that support multi-page operations
+ *</p>
+ * The selector automatically chooses the most space-efficient option based on the actual
+ * space requirements of each approach.
+ */
+public class PageZeroWriterFlavorSelector implements IColumnPageZeroWriterFlavorSelector {
+ protected byte writerFlag = IColumnPageZeroWriter.ColumnPageZeroWriterType.DEFAULT.getWriterFlag();
+
+ // Cache of writer instances to avoid repeated object creation
+ private final Byte2ObjectArrayMap<IColumnPageZeroWriter> writers;
+ private final Byte2ObjectArrayMap<IColumnPageZeroReader> readers;
+
+ public PageZeroWriterFlavorSelector() {
+ writers = new Byte2ObjectArrayMap<>();
+ readers = new Byte2ObjectArrayMap<>();
+ }
+
+ /**
+ * Selects the optimal page zero writer based on space efficiency.
+ * <p>
+ * This method compares the space requirements of both writer types and selects
+ * the one that uses less space. This optimization is particularly beneficial
+ * for sparse datasets where many columns may be null or missing.
+ * </p>
+ * @param spaceOccupiedByDefaultWriter Space in bytes required by the default writer
+ * @param spaceOccupiedBySparseWriter Space in bytes required by the sparse writer
+ */
+ @Override
+ public void switchPageZeroWriterIfNeeded(int spaceOccupiedByDefaultWriter, int spaceOccupiedBySparseWriter) {
+ if (spaceOccupiedByDefaultWriter <= spaceOccupiedBySparseWriter) {
+ // Default writer is more space-efficient (or equal), use it
+ writerFlag = MULTI_PAGE_DEFAULT_WRITER_FLAG;
+ } else {
+ // Sparse writer is more space-efficient, use it
+ writerFlag = MULTI_PAGE_SPARSE_WRITER_FLAG;
+ }
+ }
+
+ @Override
+ public void setPageZeroWriterFlag(byte writerFlag) {
+ this.writerFlag = writerFlag;
+ }
+
+ /**
+ * Returns the currently selected page zero writer instance.
+ * Writers are cached to avoid repeated object creation.
+ *
+ * @return the selected writer instance
+ * @throws IllegalStateException if an unsupported writer flag is encountered
+ */
+ @Override
+ public IColumnPageZeroWriter getPageZeroWriter(IColumnWriteMultiPageOp multiPageOpRef, int zerothSegmentMaxColumns,
+ int bufferCapacity) {
+ return switch (writerFlag) {
+ case DEFAULT_WRITER_FLAG -> writers.computeIfAbsent(DEFAULT_WRITER_FLAG, k -> new DefaultColumnPageZeroWriter());
+ case SPARSE_WRITER_FLAG -> writers.computeIfAbsent(SPARSE_WRITER_FLAG, k -> new SparseColumnPageZeroWriter());
+ case MULTI_PAGE_DEFAULT_WRITER_FLAG -> writers.computeIfAbsent(MULTI_PAGE_DEFAULT_WRITER_FLAG, k -> new DefaultColumnMultiPageZeroWriter(multiPageOpRef, zerothSegmentMaxColumns, bufferCapacity));
+ case MULTI_PAGE_SPARSE_WRITER_FLAG -> writers.computeIfAbsent(MULTI_PAGE_SPARSE_WRITER_FLAG, k -> new SparseColumnMultiPageZeroWriter(multiPageOpRef, zerothSegmentMaxColumns, bufferCapacity));
+ default -> throw new IllegalStateException("Unsupported writer flag: " + writerFlag);
+ };
+ }
+
+ @Override
+ public byte getWriterFlag() {
+ return writerFlag;
+ }
+
+ /**
+ * Creates a page zero reader instance based on the provided flag.
+ * This method is used during deserialization to create the appropriate reader
+ * for the writer type that was used during serialization.
+ *
+ * @param flag The flag code identifying the writer type (DEFAULT_WRITER_FLAG=default, SPARSE_WRITER_FLAG=sparse)
+ * @return the appropriate reader instance
+ * @throws IllegalStateException if an unsupported reader flag is encountered
+ */
+ @Override
+ public IColumnPageZeroReader createPageZeroReader(byte flag, int bufferCapacity) {
+ return switch (flag) {
+ case DEFAULT_WRITER_FLAG -> readers.computeIfAbsent(DEFAULT_WRITER_FLAG, k -> new DefaultColumnPageZeroReader());
+ case SPARSE_WRITER_FLAG -> readers.computeIfAbsent(SPARSE_WRITER_FLAG, k -> new SparseColumnPageZeroReader());
+ case MULTI_PAGE_DEFAULT_WRITER_FLAG -> readers.computeIfAbsent(MULTI_PAGE_DEFAULT_WRITER_FLAG, k -> new DefaultColumnMultiPageZeroReader(bufferCapacity));
+ case MULTI_PAGE_SPARSE_WRITER_FLAG -> readers.computeIfAbsent(MULTI_PAGE_SPARSE_WRITER_FLAG, k -> new SparseColumnMultiPageZeroReader(bufferCapacity));
+ default -> throw new IllegalStateException("Unsupported reader flag: " + flag);
+ };
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
new file mode 100644
index 0000000..d756a6c
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.readers;
+
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.FLAG_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NEXT_LEAF_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NUMBER_OF_COLUMNS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.RIGHT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.TUPLE_COUNT_OFFSET;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class DefaultColumnPageZeroReader implements IColumnPageZeroReader {
+ protected static Logger LOGGER = LogManager.getLogger();
+
+ protected ByteBuffer pageZeroBuf;
+ protected static final BitSet EMPTY_SEGMENTS = new BitSet();
+ protected int numberOfPresentColumns;
+ protected int headerSize;
+
+ public DefaultColumnPageZeroReader() {
+ }
+
+ @Override
+ public void reset(ByteBuffer pageZeroBuf, int headerSize) {
+ this.pageZeroBuf = pageZeroBuf;
+ this.numberOfPresentColumns = pageZeroBuf.getInt(NUMBER_OF_COLUMNS_OFFSET);
+ this.headerSize = headerSize;
+ }
+
+ public void reset(ByteBuffer pageZeroBuf, int numberOfPresentColumns, int headerSize) {
+ this.pageZeroBuf = pageZeroBuf;
+ this.numberOfPresentColumns = numberOfPresentColumns;
+ this.headerSize = headerSize;
+ }
+
+ @Override
+ public int getColumnOffset(int columnIndex) {
+ return pageZeroBuf.getInt(headerSize + columnIndex * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE);
+ }
+
+ protected int getColumnFilterOffset(int columnIndex) {
+ int columnsOffsetEnd = headerSize + numberOfPresentColumns * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ return columnsOffsetEnd + columnIndex * DefaultColumnPageZeroWriter.FILTER_SIZE;
+ }
+
+ @Override
+ public long getColumnFilterMin(int columnIndex) {
+ int filterOffset = getColumnFilterOffset(columnIndex);
+ return pageZeroBuf.getLong(filterOffset);
+ }
+
+ @Override
+ public long getColumnFilterMax(int columnIndex) {
+ int filterOffset = getColumnFilterOffset(columnIndex);
+ return pageZeroBuf.getLong(filterOffset + Long.BYTES);
+ }
+
+ @Override
+ public void skipFilters() {
+ int filterEndOffset = getColumnFilterOffset(numberOfPresentColumns);
+ pageZeroBuf.position(filterEndOffset);
+ }
+
+ @Override
+ public void skipColumnOffsets() {
+ int columnEndOffset = headerSize + numberOfPresentColumns * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ pageZeroBuf.position(columnEndOffset);
+ }
+
+ @Override
+ public int getTupleCount() {
+ return pageZeroBuf.getInt(TUPLE_COUNT_OFFSET);
+ }
+
+ @Override
+ public int getNumberOfPageZeroSegments() {
+ return 1;
+ }
+
+ @Override
+ public int getLeftMostKeyOffset() {
+ return pageZeroBuf.getInt(LEFT_MOST_KEY_OFFSET);
+ }
+
+ @Override
+ public int getRightMostKeyOffset() {
+ return pageZeroBuf.getInt(RIGHT_MOST_KEY_OFFSET);
+ }
+
+ @Override
+ public int getNumberOfPresentColumns() {
+ return pageZeroBuf.getInt(NUMBER_OF_COLUMNS_OFFSET);
+ }
+
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) {
+ return columnIndex;
+ }
+
+ @Override
+ public int getNextLeaf() {
+ return pageZeroBuf.getInt(NEXT_LEAF_OFFSET);
+ }
+
+ @Override
+ public int getMegaLeafNodeLengthInBytes() {
+ return pageZeroBuf.getInt(MEGA_LEAF_NODE_LENGTH);
+ }
+
+ @Override
+ public int getPageZeroCapacity() {
+ return pageZeroBuf.capacity();
+ }
+
+ @Override
+ public boolean isValidColumn(int columnIndex) {
+ return columnIndex < numberOfPresentColumns;
+ }
+
+ @Override
+ public void getAllColumns(BitSet presentColumns) {
+ int numberOfColumns = numberOfPresentColumns;
+ presentColumns.set(0, numberOfColumns);
+ }
+
+ @Override
+ public ByteBuffer getPageZeroBuf() {
+ return pageZeroBuf;
+ }
+
+ @Override
+ public int populateOffsetColumnIndexPairs(long[] offsetColumnIndexPairs) {
+ int columnOffsetStart = headerSize;
+ for (int i = 0; i < numberOfPresentColumns; i++) {
+ int offset = pageZeroBuf.getInt(columnOffsetStart);
+ offsetColumnIndexPairs[i] = IntPairUtil.of(offset, i);
+ columnOffsetStart += DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+ return numberOfPresentColumns;
+ }
+
+ @Override
+ public BitSet getPageZeroSegmentsPages() {
+ return EMPTY_SEGMENTS;
+ }
+
+ @Override
+ public int getHeaderSize() {
+ return headerSize;
+ }
+
+ @Override
+ public void resetStream(IColumnBufferProvider pageZeroSegmentBufferProvider) {
+ throw new UnsupportedOperationException("Not supported for DefaultColumnPageZeroReader");
+ }
+
+ @Override
+ public BitSet markRequiredPageSegments(BitSet projectedColumns, int pageZeroId, boolean markAll) {
+ return EMPTY_SEGMENTS;
+ }
+
+ @Override
+ public void unPinNotRequiredPageZeroSegments() throws HyracksDataException {
+ // No-OP
+ }
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("numberOfPresentColumns", numberOfPresentColumns);
+ readerNode.put("flag", pageZeroBuf.get(FLAG_OFFSET));
+ LOGGER.debug("SingleColumnPageZeroReader Info: {}", readerNode.toPrettyString());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/SparseColumnPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/SparseColumnPageZeroReader.java
new file mode 100644
index 0000000..5955d5e
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/SparseColumnPageZeroReader.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.readers;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.asterix.column.zero.writers.SparseColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+
+public class SparseColumnPageZeroReader extends DefaultColumnPageZeroReader {
+ private final Int2IntOpenHashMap columnIndexToRelativeColumnIndex;
+
+ public SparseColumnPageZeroReader() {
+ columnIndexToRelativeColumnIndex = new Int2IntOpenHashMap();
+ columnIndexToRelativeColumnIndex.defaultReturnValue(-1);
+ }
+
+ @Override
+ public void reset(ByteBuffer pageZeroBuf, int headerSize) {
+ super.reset(pageZeroBuf, headerSize);
+ columnIndexToRelativeColumnIndex.clear();
+ }
+
+ @Override
+ public void reset(ByteBuffer pageZeroBuf, int numberOfPresentColumns, int headerSize) {
+ super.reset(pageZeroBuf, numberOfPresentColumns, headerSize);
+ columnIndexToRelativeColumnIndex.clear();
+ }
+
+ @Override
+ public int getColumnOffset(int columnIndex) {
+ int relativeColumnIndex = getRelativeColumnIndex(columnIndex);
+ return pageZeroBuf.getInt(
+ headerSize + relativeColumnIndex * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE + Integer.BYTES);
+ }
+
+ @Override
+ public int getColumnFilterOffset(int columnIndex) {
+ int relativeColumnIndex = getRelativeColumnIndex(columnIndex);
+ int columnsOffsetEnd = headerSize + numberOfPresentColumns * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ return columnsOffsetEnd + relativeColumnIndex * DefaultColumnPageZeroWriter.FILTER_SIZE;
+ }
+
+ @Override
+ public void skipFilters() {
+ int filterEndOffset = headerSize + numberOfPresentColumns
+ * (SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE + SparseColumnPageZeroWriter.FILTER_SIZE);
+ pageZeroBuf.position(filterEndOffset);
+ }
+
+ @Override
+ public void skipColumnOffsets() {
+ int columnsOffsetEnd = headerSize + numberOfPresentColumns * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ pageZeroBuf.position(columnsOffsetEnd);
+ }
+
+ // x + 0, 8, 16, .... , 8*(n-1)
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) {
+ // if the entry is in cache, return it
+ int cachedIndex = columnIndexToRelativeColumnIndex.get(columnIndex);
+ if (cachedIndex != -1) {
+ return cachedIndex;
+ }
+ int startColumnIndex = getColumnIndex(0);
+ int startColumn = pageZeroBuf.getInt(startColumnIndex);
+ if (startColumn == columnIndex) {
+ columnIndexToRelativeColumnIndex.put(columnIndex, 0);
+ return 0;
+ }
+
+ int totalColumns = numberOfPresentColumns;
+ int lastColumnIndex = getColumnIndex(totalColumns - 1);
+ int lastColumn = pageZeroBuf.getInt(lastColumnIndex);
+ if (lastColumn == columnIndex) {
+ columnIndexToRelativeColumnIndex.put(columnIndex, totalColumns - 1);
+ return totalColumns - 1;
+ }
+
+ int start = 0;
+ int end = totalColumns - 1;
+ while (start <= end) {
+ int mid = start + (end - start) / 2;
+ int midIndex = getColumnIndex(mid);
+ int midColumnIndex = pageZeroBuf.getInt(midIndex);
+ if (midColumnIndex == columnIndex) {
+ columnIndexToRelativeColumnIndex.put(columnIndex, mid);
+ return mid; // this is the relative index
+ } else if (midColumnIndex < columnIndex) {
+ start = mid + 1;
+ } else {
+ end = mid - 1;
+ }
+ }
+
+ return -1;
+ }
+
+ private int getColumnIndex(int index) {
+ return headerSize + index * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+
+ @Override
+ public boolean isValidColumn(int columnIndex) {
+ int relativeColumnIndex = getRelativeColumnIndex(columnIndex);
+ return relativeColumnIndex != -1;
+ }
+
+ @Override
+ public void getAllColumns(BitSet presentColumns) {
+ if (numberOfPresentColumns == 0) {
+ return;
+ }
+
+ int columnIndex = headerSize;
+ int limit = columnIndex + numberOfPresentColumns * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+
+ while (columnIndex < limit) {
+ int column = pageZeroBuf.getInt(columnIndex);
+ presentColumns.set(column);
+ columnIndex += SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+ }
+
+ @Override
+ public int populateOffsetColumnIndexPairs(long[] offsetColumnIndexPairs) {
+ int columnIndex = getColumnIndex(0);
+ for (int i = 0; i < numberOfPresentColumns; i++) {
+ int column = pageZeroBuf.getInt(columnIndex);
+ int offset = pageZeroBuf.getInt(columnIndex + SparseColumnPageZeroWriter.COLUMN_INDEX_SIZE);
+ offsetColumnIndexPairs[i] = IntPairUtil.of(offset, column);
+ columnIndex += SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+ return numberOfPresentColumns;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/DefaultColumnPageZeroWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/DefaultColumnPageZeroWriter.java
new file mode 100644
index 0000000..e7d4e66
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/DefaultColumnPageZeroWriter.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.writers;
+
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.FLAG_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.HEADER_SIZE;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NUMBER_OF_COLUMNS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.RIGHT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.SIZE_OF_COLUMNS_OFFSETS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.TUPLE_COUNT_OFFSET;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.asterix.column.bytes.stream.out.ByteBufferOutputStream;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IValuesWriter;
+
+/**
+ * Default implementation of page zero writer that allocates space for all columns in the schema.
+ * <p>
+ * This writer uses a fixed layout where every column in the schema has a reserved slot,
+ * regardless of whether data is present for that column. This approach is optimal for
+ * dense datasets where most columns contain data.
+ * <p>
+ * Memory layout in page zero:
+ * 1. Column offsets: 4 bytes per column (numberOfColumns * 4 bytes)
+ * 2. Column filters: 16 bytes per column (numberOfColumns * 16 bytes) - min/max values
+ * 3. Primary key data: variable size, written sequentially
+ */
+public class DefaultColumnPageZeroWriter implements IColumnPageZeroWriter {
+ /** Size in bytes for storing a column offset */
+ public static final int COLUMN_OFFSET_SIZE = Integer.BYTES;
+ /** Size in bytes for storing column filter (min + max values) */
+ public static final int FILTER_SIZE = Long.BYTES * 2; // min and max
+
+ protected final ByteBufferOutputStream primaryKeys;
+ protected ByteBuffer pageZero;
+ protected int headerSize;
+ private int numberOfColumns;
+
+ // Offset positions within page zero buffer
+ protected int primaryKeysOffset; // Where primary key data starts
+ protected int columnsOffset; // Where column offset array starts
+ protected int filtersOffset; // Where column filter array starts
+
+ public DefaultColumnPageZeroWriter() {
+ primaryKeys = new ByteBufferOutputStream();
+ }
+
+ @Override
+ public void resetBasedOnColumns(int[] presentColumns, int numberOfColumns, int headerSize) {
+ this.numberOfColumns = numberOfColumns;
+ primaryKeysOffset = headerSize;
+ this.headerSize = headerSize;
+ pageZero.position(headerSize);
+ }
+
+ @Override
+ public byte flagCode() {
+ return DEFAULT_WRITER_FLAG;
+ }
+
+ /**
+ * Allocates space in page zero for all column metadata.
+ * <p>
+ * The allocation strategy reserves space for all columns in the schema:
+ * - Column offsets: Fixed array of 4-byte integers
+ * - Column filters: Fixed array of 16-byte min/max pairs
+ * - Primary keys: Variable-size data written after metadata
+ */
+ @Override
+ public void allocateColumns() {
+ // allocate space for columns' offset (4 * numberOfColumns)
+ columnsOffset = primaryKeysOffset;
+ primaryKeysOffset += COLUMN_OFFSET_SIZE * numberOfColumns;
+
+ // allocate space for columns' filter (8 + 8) * numberOfColumns
+ filtersOffset = primaryKeysOffset;
+ primaryKeysOffset += FILTER_SIZE * numberOfColumns;
+
+ // reset the position for pageZero,
+ // the primary keys will be written from this offset
+ pageZero.position(primaryKeysOffset);
+ primaryKeys.reset(pageZero);
+ }
+
+ /**
+ * Records a column's data offset using direct array indexing.
+ * In the default layout, the column index directly maps to the array position.
+ *
+ * @param absoluteColumnIndex The absolute column index (unused in default layout)
+ * @param relativeColumnIndex The column index used for array positioning
+ * @param offset The byte offset where the column's data begins
+ */
+ @Override
+ public void putColumnOffset(int absoluteColumnIndex, int relativeColumnIndex, int offset) {
+ pageZero.putInt(columnsOffset + COLUMN_OFFSET_SIZE * relativeColumnIndex, offset);
+ }
+
+ /**
+ * Stores column filter information using direct array indexing.
+ * Filters enable efficient column pruning during query execution.
+ *
+ * @param relativeColumnIndex The column index used for array positioning
+ * @param normalizedMinValue The normalized minimum value in the column
+ * @param normalizedMaxValue The normalized maximum value in the column
+ */
+ @Override
+ public void putColumnFilter(int relativeColumnIndex, long normalizedMinValue, long normalizedMaxValue) {
+ int offset = filtersOffset + relativeColumnIndex * FILTER_SIZE;
+ pageZero.putLong(offset, normalizedMinValue);
+ pageZero.putLong(offset + Long.BYTES, normalizedMaxValue);
+ }
+
+ /**
+ * Writes primary key columns directly to page zero.
+ * Primary keys are stored in page zero for fast access during operations.
+ *
+ * @param primaryKeyWriters Array of writers containing primary key data
+ * @throws HyracksDataException If an error occurs during writing
+ */
+ @Override
+ public void writePrimaryKeyColumns(IValuesWriter[] primaryKeyWriters) throws HyracksDataException {
+ for (int i = 0; i < primaryKeyWriters.length; i++) {
+ IValuesWriter writer = primaryKeyWriters[i];
+ // Record the offset where this primary key column starts
+ putColumnOffset(i, i, primaryKeysOffset + primaryKeys.size());
+ // Write the actual primary key data
+ writer.flush(primaryKeys);
+ }
+ }
+
+ @Override
+ public void setPageZero(ByteBuffer pageZero) {
+ // this method is used to set the pageZero buffer
+ // only caller is the MultiColumnPageZeroWriter
+ this.pageZero = pageZero;
+ }
+
+ public void flush(ByteBuffer buf, int numberOfTuples, ITupleReference minKey, ITupleReference maxKey,
+ AbstractColumnTupleWriter columnWriter, ITreeIndexTupleWriter rowTupleWriter) throws HyracksDataException {
+ this.pageZero = buf;
+ // Prepare the space for writing the columns' information such as the primary keys
+ pageZero.position(HEADER_SIZE);
+ this.primaryKeysOffset = buf.position();
+ // Flush the columns to persistence pages and write the length of the mega leaf node in pageZero
+ pageZero.putInt(MEGA_LEAF_NODE_LENGTH, columnWriter.flush(this));
+ // Write min and max keys
+ int offset = buf.position();
+ buf.putInt(LEFT_MOST_KEY_OFFSET, offset);
+ offset += rowTupleWriter.writeTuple(minKey, buf.array(), offset);
+ buf.putInt(RIGHT_MOST_KEY_OFFSET, offset);
+ rowTupleWriter.writeTuple(maxKey, buf.array(), offset);
+
+ // Write page information
+ buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples);
+ buf.put(FLAG_OFFSET, flagCode());
+ buf.putInt(NUMBER_OF_COLUMNS_OFFSET, getNumberOfColumns());
+ buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, getColumnOffsetsSize());
+
+ // reset the collected meta info
+ columnWriter.reset();
+ }
+
+ public void flush(ByteBuffer buf, int numberOfTuples, AbstractColumnTupleWriter writer)
+ throws HyracksDataException {
+ this.pageZero = buf;
+ pageZero.position(HEADER_SIZE);
+ this.primaryKeysOffset = buf.position();
+ pageZero.putInt(MEGA_LEAF_NODE_LENGTH, writer.flush(this));
+ buf.putInt(NUMBER_OF_COLUMNS_OFFSET, getNumberOfColumns());
+ buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples);
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return numberOfColumns;
+ }
+
+ /**
+ * In the default layout, all columns are always included since space is pre-allocated.
+ *
+ * @param presentColumns Set of columns present in this page (unused)
+ * @param columnIndex The column index to check (unused)
+ * @param includeChildrenColumns Whether to include child columns (unused)
+ * @return always true for default layout
+ */
+ @Override
+ public boolean includeOrderedColumn(BitSet presentColumns, int columnIndex, boolean includeChildrenColumns) {
+ return true;
+ }
+
+ @Override
+ public int getPageZeroBufferCapacity() {
+ return pageZero.capacity();
+ }
+
+ /**
+ * In the default layout, the relative index is the same as the absolute index.
+ *
+ * @param columnIndex The absolute column index
+ * @return the same column index (identity mapping)
+ */
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) {
+ return columnIndex;
+ }
+
+ @Override
+ public int getColumnOffsetsSize() {
+ return numberOfColumns * COLUMN_OFFSET_SIZE;
+ }
+
+ @Override
+ public int getHeaderSize() {
+ return headerSize;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/SparseColumnPageZeroWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/SparseColumnPageZeroWriter.java
new file mode 100644
index 0000000..4fc2550
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/SparseColumnPageZeroWriter.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.writers;
+
+import static org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter.FILTER_SIZE;
+
+import java.util.BitSet;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IValuesWriter;
+
+/**
+ * Sparse implementation of page zero writer that only allocates space for present columns.
+ * <p>
+ * This writer optimizes space usage for sparse datasets by storing only the columns
+ * that actually contain data. Each column entry includes both the column index and
+ * its data offset, allowing for efficient lookup while minimizing space overhead.
+ *<p>
+ * Memory layout in page zero:
+ * 1. Column entries: 8 bytes per present column (4 bytes index + 4 bytes offset)
+ * 2. Column filters: 16 bytes per present column (min/max values)
+ * 3. Primary key data: variable size, written sequentially
+ * <p>
+ * This layout is particularly beneficial when the number of present columns is
+ * significantly smaller than the total schema size.
+ */
+public class SparseColumnPageZeroWriter extends DefaultColumnPageZeroWriter {
+ /** Size in bytes for storing a column index */
+ public static final int COLUMN_INDEX_SIZE = Integer.BYTES;
+ /** Size in bytes for storing a column entry (index + offset) */
+ public static final int COLUMN_OFFSET_SIZE = Integer.BYTES + COLUMN_INDEX_SIZE;
+
+ private int[] presentColumns;
+ private int numberOfPresentColumns;
+
+ public SparseColumnPageZeroWriter() {
+ super();
+ }
+
+ @Override
+ public void resetBasedOnColumns(int[] presentColumns, int numberOfColumns /* not being used */, int headerSize) {
+ this.presentColumns = presentColumns;
+ this.numberOfPresentColumns = presentColumns.length;
+ this.primaryKeysOffset = headerSize;
+ this.headerSize = headerSize;
+ pageZero.position(headerSize);
+ }
+
+ public void resetInnerBasedOnColumns(int[] presentColumns, int numberOfPresentColumns, int headerSize) {
+ this.presentColumns = presentColumns;
+ this.numberOfPresentColumns = numberOfPresentColumns;
+ this.primaryKeysOffset = headerSize; // Reset primary keys offset for sparse layout
+ pageZero.position(headerSize);
+ }
+
+ @Override
+ public byte flagCode() {
+ return SPARSE_WRITER_FLAG;
+ }
+
+ /**
+ * Allocates space in page zero for present column metadata only.
+ *
+ * The allocation strategy reserves space only for columns that contain data:
+ * - Column entries: Array of (index, offset) pairs for present columns
+ * - Column filters: Array of min/max pairs for present columns only
+ * - Primary keys: Variable-size data written after metadata
+ */
+ @Override
+ public void allocateColumns() {
+ // allocate space for columns' offset (8 * numberOfPresentColumns)
+ columnsOffset = primaryKeysOffset;
+ primaryKeysOffset += COLUMN_OFFSET_SIZE * numberOfPresentColumns;
+
+ // allocate space for filters'
+ filtersOffset = primaryKeysOffset;
+ primaryKeysOffset += FILTER_SIZE * numberOfPresentColumns;
+
+ // reset the position for pageZero,
+ // the primary keys will be written from this offset
+ pageZero.position(primaryKeysOffset);
+ primaryKeys.reset(pageZero);
+ }
+
+ /**
+ * Records a column's data offset along with its absolute column index.
+ *
+ * In the sparse layout, each entry stores both the original column index
+ * and the data offset, enabling lookup of sparse columns.
+ *
+ * @param absoluteColumnIndex The absolute column index in the schema
+ * @param relativeColumnIndex The position within the present columns array
+ * @param offset The byte offset where the column's data begins
+ */
+ @Override
+ public void putColumnOffset(int absoluteColumnIndex, int relativeColumnIndex, int offset) {
+ int columnOffset = columnsOffset + COLUMN_OFFSET_SIZE * relativeColumnIndex;
+ // Store the absolute column index first
+ pageZero.putInt(columnOffset, absoluteColumnIndex);
+ // Then store the data offset
+ pageZero.putInt(columnOffset + Integer.BYTES, offset);
+ }
+
+ /**
+ * Stores column filter information for present columns only.
+ * Uses the relative column index to position within the sparse filter array.
+ *
+ * @param relativeColumnIndex The position within the present columns array
+ * @param normalizedMinValue The normalized minimum value in the column
+ * @param normalizedMaxValue The normalized maximum value in the column
+ */
+ @Override
+ public void putColumnFilter(int relativeColumnIndex, long normalizedMinValue, long normalizedMaxValue) {
+ int offset = filtersOffset + relativeColumnIndex * FILTER_SIZE;
+ pageZero.putLong(offset, normalizedMinValue);
+ pageZero.putLong(offset + Long.BYTES, normalizedMaxValue);
+ }
+
+ /**
+ * Writes primary key columns directly to page zero.
+ * Primary keys are always present and stored similarly to the default layout.
+ *
+ * @param primaryKeyWriters Array of writers containing primary key data
+ * @throws HyracksDataException If an error occurs during writing
+ */
+ @Override
+ public void writePrimaryKeyColumns(IValuesWriter[] primaryKeyWriters) throws HyracksDataException {
+ for (int i = 0; i < primaryKeyWriters.length; i++) {
+ IValuesWriter writer = primaryKeyWriters[i];
+ // Record the offset where this primary key column starts
+ putColumnOffset(i, i, primaryKeysOffset + primaryKeys.size());
+ // Write the actual primary key data
+ writer.flush(primaryKeys);
+ }
+ }
+
+ /**
+ * Performs binary search to find the relative position of a column index
+ * within the sorted present columns array.
+ *
+ * @param columnIndex The absolute column index to find
+ * @return the relative position within present columns, or -1 if not found
+ */
+ public int findColumnIndex(int[] presentColumns, int numberOfPresentColumns, int columnIndex) {
+ int low = 0;
+ int high = numberOfPresentColumns - 1;
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ int midVal = presentColumns[mid];
+ if (midVal == columnIndex) {
+ return mid;
+ } else if (midVal < columnIndex) {
+ low = mid + 1;
+ } else {
+ high = mid - 1;
+ }
+ }
+
+ return -1;
+ }
+
+ /**
+ * Determines whether a column should be included based on presence in the sparse set.
+ *
+ * For sparse layouts, only explicitly present columns or child columns
+ * (when includeChildrenColumns is true) are included.
+ *
+ * @param presentColumns Set of columns present in this page
+ * @param columnIndex The column index to check
+ * @param includeChildrenColumns Whether to include child columns for complex types
+ * @return true if the column should be included
+ */
+ @Override
+ public boolean includeOrderedColumn(BitSet presentColumns, int columnIndex, boolean includeChildrenColumns) {
+ return includeChildrenColumns || presentColumns.get(columnIndex);
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return numberOfPresentColumns;
+ }
+
+ /**
+ * Maps an absolute column index to its relative position within the present columns array.
+ *
+ * This mapping is essential for sparse layouts where the storage position
+ * differs from the schema position.
+ *
+ * @param columnIndex The absolute column index in the schema
+ * @return the relative position within the present columns array
+ * @throws IllegalStateException if the column index is not found in present columns
+ */
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) {
+ int columnRelativeIndex = findColumnIndex(presentColumns, numberOfPresentColumns, columnIndex);
+ if (columnRelativeIndex == -1) {
+ throw new IllegalStateException("Column index " + columnIndex + " does not exist in present columns.");
+ }
+ return columnRelativeIndex;
+ }
+
+ @Override
+ public int getColumnOffsetsSize() {
+ return numberOfPresentColumns * COLUMN_OFFSET_SIZE;
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
new file mode 100644
index 0000000..b0fc721
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.writers.multipage;
+
+import org.apache.asterix.column.bytes.stream.in.MultiPageZeroByteBuffersReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public abstract class AbstractColumnMultiPageZeroReader implements IColumnPageZeroReader {
+ protected static final Logger LOGGER = LogManager.getLogger();
+ protected MultiPageZeroByteBuffersReader segmentBuffers;
+
+ AbstractColumnMultiPageZeroReader() {
+ segmentBuffers = new MultiPageZeroByteBuffersReader();
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
new file mode 100644
index 0000000..d4ffbb4
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.writers.multipage;
+
+import static org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter.EXTENDED_HEADER_SIZE;
+import static org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter.MAX_COLUMNS_IN_ZEROTH_SEGMENT;
+import static org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter.NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NEXT_LEAF_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NUMBER_OF_COLUMNS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.RIGHT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.TUPLE_COUNT_OFFSET;
+
+import java.io.EOFException;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.asterix.column.zero.readers.DefaultColumnPageZeroReader;
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class DefaultColumnMultiPageZeroReader extends AbstractColumnMultiPageZeroReader {
+ public static final int headerSize = DefaultColumnMultiPageZeroWriter.EXTENDED_HEADER_SIZE;
+ private final DefaultColumnPageZeroReader zerothSegmentReader;
+
+ private final int maxNumberOfColumnsInAPage;
+ private final BitSet pageZeroSegmentsPages;
+ private int zerothSegmentMaxColumns;
+ private int numberOfPageZeroSegments; // includes the zeroth segment
+ private ByteBuffer pageZeroBuf;
+
+ private final VoidPointable offsetPointable;
+
+ public DefaultColumnMultiPageZeroReader(int bufferCapacity) {
+ super();
+ zerothSegmentReader = new DefaultColumnPageZeroReader();
+ this.pageZeroSegmentsPages = new BitSet();
+ this.maxNumberOfColumnsInAPage =
+ DefaultColumnMultiPageZeroWriter.getMaximumNumberOfColumnsInAPage(bufferCapacity);
+ this.offsetPointable = new VoidPointable();
+ }
+
+ @Override
+ public void resetStream(IColumnBufferProvider pageZeroSegmentBufferProvider) throws HyracksDataException {
+ segmentBuffers.reset(pageZeroSegmentBufferProvider);
+ }
+
+ @Override
+ public void reset(ByteBuffer pageZeroBuf) {
+ this.pageZeroBuf = pageZeroBuf;
+ zerothSegmentMaxColumns = pageZeroBuf.getInt(MAX_COLUMNS_IN_ZEROTH_SEGMENT);
+ zerothSegmentReader.reset(pageZeroBuf, Math.min(zerothSegmentMaxColumns, getNumberOfPresentColumns()),
+ headerSize);
+ numberOfPageZeroSegments = pageZeroBuf.getInt(NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET);
+ }
+
+ @Override
+ public void reset(ByteBuffer pageZeroBuf, int headerSize) {
+ throw new UnsupportedOperationException("This method should not be called for multi-page readers.");
+ }
+
+ @Override
+ public int getColumnOffset(int columnIndex) throws HyracksDataException {
+ try {
+ if (columnIndex < zerothSegmentMaxColumns) {
+ return zerothSegmentReader.getColumnOffset(columnIndex);
+ } else {
+ int segmentIndex = (columnIndex - zerothSegmentMaxColumns) / maxNumberOfColumnsInAPage;
+ int columnIndexInRequiredSegment = (columnIndex - zerothSegmentMaxColumns) % maxNumberOfColumnsInAPage;
+ int segmentOffset = columnIndexInRequiredSegment * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ segmentBuffers.read(segmentIndex, offsetPointable, segmentOffset,
+ DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE);
+ return IntegerPointable.getInteger(offsetPointable.getByteArray(), offsetPointable.getStartOffset());
+ }
+ } catch (EOFException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public int getNumberOfPageZeroSegments() {
+ return numberOfPageZeroSegments;
+ }
+
+ @Override
+ public BitSet getPageZeroSegmentsPages() {
+ //If pageZeroSegmentsPages is null, it means that the CloudReadContext is not being used.
+ // which indicates all the segments are being read.
+ return pageZeroSegmentsPages;
+ }
+
+ @Override
+ public long getColumnFilterMin(int columnIndex) throws HyracksDataException {
+ try {
+ if (columnIndex < zerothSegmentMaxColumns) {
+ return zerothSegmentReader.getColumnFilterMin(columnIndex);
+ } else {
+ int segmentIndex = (columnIndex - zerothSegmentMaxColumns) / maxNumberOfColumnsInAPage;
+ int columnIndexInRequiredSegment = (columnIndex - zerothSegmentMaxColumns) % maxNumberOfColumnsInAPage;
+ int segmentOffset =
+ findNumberOfColumnsInSegment(segmentIndex) * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE
+ + columnIndexInRequiredSegment * DefaultColumnPageZeroWriter.FILTER_SIZE;
+ segmentBuffers.read(segmentIndex, offsetPointable, segmentOffset, Long.BYTES);
+ return LongPointable.getLong(offsetPointable.getByteArray(), offsetPointable.getStartOffset());
+ }
+ } catch (EOFException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public long getColumnFilterMax(int columnIndex) throws HyracksDataException {
+ try {
+ if (columnIndex < zerothSegmentMaxColumns) {
+ return zerothSegmentReader.getColumnFilterMax(columnIndex);
+ } else {
+ int segmentIndex = (columnIndex - zerothSegmentMaxColumns) / maxNumberOfColumnsInAPage;
+ int columnIndexInRequiredSegment = (columnIndex - zerothSegmentMaxColumns) % maxNumberOfColumnsInAPage;
+ int segmentOffset =
+ findNumberOfColumnsInSegment(segmentIndex) * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE
+ + columnIndexInRequiredSegment * DefaultColumnPageZeroWriter.FILTER_SIZE;
+ segmentOffset += Long.BYTES; // Move to the max value in the filter
+ segmentBuffers.read(segmentIndex, offsetPointable, segmentOffset, Long.BYTES);
+ return LongPointable.getLong(offsetPointable.getByteArray(), offsetPointable.getStartOffset());
+ }
+ } catch (EOFException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private int findNumberOfColumnsInSegment(int segmentIndex) {
+ // starts from 1st segment, not from 0th segment
+ if (segmentIndex == numberOfPageZeroSegments - 2) {
+ return getNumberOfPresentColumns() - zerothSegmentMaxColumns
+ - (numberOfPageZeroSegments - 2) * maxNumberOfColumnsInAPage;
+ }
+ // For segments beyond the zeroth segment, we can have maximum number of columns in a page, except the last segment.
+ return maxNumberOfColumnsInAPage;
+ }
+
+ @Override
+ public void skipFilters() {
+ zerothSegmentReader.skipFilters();
+ }
+
+ @Override
+ public void skipColumnOffsets() {
+ zerothSegmentReader.skipColumnOffsets();
+ }
+
+ @Override
+ public int getTupleCount() {
+ return pageZeroBuf.getInt(TUPLE_COUNT_OFFSET);
+ }
+
+ @Override
+ public int getLeftMostKeyOffset() {
+ return pageZeroBuf.getInt(LEFT_MOST_KEY_OFFSET);
+ }
+
+ @Override
+ public int getRightMostKeyOffset() {
+ return pageZeroBuf.getInt(RIGHT_MOST_KEY_OFFSET);
+ }
+
+ @Override
+ public int getNumberOfPresentColumns() {
+ return pageZeroBuf.getInt(NUMBER_OF_COLUMNS_OFFSET);
+ }
+
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) {
+ return columnIndex;
+ }
+
+ @Override
+ public int getNextLeaf() {
+ return pageZeroBuf.getInt(NEXT_LEAF_OFFSET);
+ }
+
+ @Override
+ public int getMegaLeafNodeLengthInBytes() {
+ return pageZeroBuf.getInt(MEGA_LEAF_NODE_LENGTH);
+ }
+
+ @Override
+ public int getPageZeroCapacity() {
+ return pageZeroBuf.capacity();
+ }
+
+ @Override
+ public boolean isValidColumn(int columnIndex) {
+ return columnIndex < getNumberOfPresentColumns();
+ }
+
+ @Override
+ public void getAllColumns(BitSet presentColumns) {
+ int numberOfColumns = getNumberOfPresentColumns();
+ presentColumns.set(0, numberOfColumns);
+ }
+
+ @Override
+ public ByteBuffer getPageZeroBuf() {
+ return pageZeroBuf;
+ }
+
+ @Override
+ public int populateOffsetColumnIndexPairs(long[] offsetColumnIndexPairs) {
+ int columnOffsetStart = headerSize;
+ int numberOfColumns = getNumberOfPresentColumns();
+ int currentColumnIndex = 0;
+ while (currentColumnIndex < Math.min(numberOfColumns, zerothSegmentMaxColumns)) {
+ // search in the 0th segment
+ int offset = pageZeroBuf.getInt(columnOffsetStart);
+ offsetColumnIndexPairs[currentColumnIndex] = IntPairUtil.of(offset, currentColumnIndex);
+ columnOffsetStart += DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ currentColumnIndex++;
+ }
+
+ if (numberOfColumns > zerothSegmentMaxColumns) {
+ // read the rest of the columns from the segment stream
+ currentColumnIndex = segmentBuffers.readOffset(offsetColumnIndexPairs, zerothSegmentMaxColumns,
+ maxNumberOfColumnsInAPage, currentColumnIndex);
+ }
+ return currentColumnIndex;
+ }
+
+ @Override
+ public BitSet markRequiredPageSegments(BitSet projectedColumns, int pageZeroId, boolean markAll) {
+ pageZeroSegmentsPages.clear();
+ // Not marking the zeroth segment
+ if (numberOfPageZeroSegments == 1 || markAll) {
+ // mark all segments as required
+ pageZeroSegmentsPages.set(1, numberOfPageZeroSegments);
+ } else {
+ // Iterate over the projected columns and mark the segments that contain them
+ int currentIndex = projectedColumns.nextSetBit(zerothSegmentMaxColumns);
+ int totalNumberOfColumns = getNumberOfPresentColumns();
+ while (currentIndex >= 0 && currentIndex < totalNumberOfColumns) {
+ int rangeEnd = projectedColumns.nextClearBit(currentIndex); // exclusive
+
+ int fromSegmentIndex = (currentIndex - zerothSegmentMaxColumns) / maxNumberOfColumnsInAPage + 1;
+ int toSegmentIndex = (rangeEnd - 1 - zerothSegmentMaxColumns) / maxNumberOfColumnsInAPage + 1;
+
+ if (fromSegmentIndex <= toSegmentIndex) {
+ pageZeroSegmentsPages.set(fromSegmentIndex, toSegmentIndex + 1); // inclusive range
+ }
+
+ currentIndex = projectedColumns.nextSetBit(rangeEnd);
+ }
+ }
+
+ return pageZeroSegmentsPages;
+ }
+
+ @Override
+ public void unPinNotRequiredPageZeroSegments() throws HyracksDataException {
+ segmentBuffers.unPinNotRequiredSegments(pageZeroSegmentsPages, numberOfPageZeroSegments);
+ }
+
+ @Override
+ public int getHeaderSize() {
+ return EXTENDED_HEADER_SIZE;
+ }
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("maxColumnsInZerothSegment", zerothSegmentMaxColumns);
+ readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage);
+ readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments);
+ LOGGER.debug("DefaultColumnMultiPageZeroReader Info: {}", readerNode.toPrettyString());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroWriter.java
new file mode 100644
index 0000000..5c7d383
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroWriter.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.writers.multipage;
+
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.FLAG_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.HEADER_SIZE;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NUMBER_OF_COLUMNS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.RIGHT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.SIZE_OF_COLUMNS_OFFSETS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.TUPLE_COUNT_OFFSET;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.asterix.column.bytes.stream.out.MultiPersistentPageZeroBufferBytesOutputStream;
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IValuesWriter;
+
+/*
+[ PageZero Segment 0 ]
+──────────────────────────────────────────────────────────────────────────────
+| Headers |
+| ───────────────────────────────────────────────────────────────────────── |
+| TupleCountOffset |
+| MaxColumnsInZerothSegment |
+| LevelOffset |
+| NumberOfColumnsOffset |
+| LeftMostKeyOffset |
+| RightMostKeyOffset |
+| SizeOfColumnsOffsetsOffset |
+| MegaLeafNodeLength |
+| FlagOffset |
+| NextLeafOffset |
+| NumberOfPageSegments |
+| MaxColumnsInPageZeroSegment |
+
+| Min Primary Key |
+| Max Primary Key |
+| Primary Key Values |
+| [ offset₁, min₁, max₁ ] |
+| [ offset₂, min₂, max₂ ] |
+| [ offset₃, min₃, max₃ ] |
+| ... |
+
+[ PageZero Segment 1..N ]
+──────────────────────────────────────────────────────────────────────────────
+| Additional column metadata (same format) |
+*/
+public class DefaultColumnMultiPageZeroWriter implements IColumnPageZeroWriter {
+ // for storing max columns allowed in zeroth segment
+ public static final int NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET = HEADER_SIZE;
+ public static final int MAX_COLUMNS_IN_ZEROTH_SEGMENT = HEADER_SIZE + Integer.BYTES;
+ public static final int EXTENDED_HEADER_SIZE = MAX_COLUMNS_IN_ZEROTH_SEGMENT + Integer.BYTES;
+
+ private final MultiPersistentPageZeroBufferBytesOutputStream segments;
+ private final DefaultColumnPageZeroWriter zerothSegmentWriter;
+ // maximum number of columns that can be laid out in the zeroth segments
+ private final int zerothSegmentMaxColumns;
+ private final int maximumNumberOfColumnsInAPage; // this is the maximum number of columns that can be laid out in a page
+
+ private int numberOfColumns;
+ private int numberOfColumnInZerothSegment;
+ private int numberOfPageZeroSegments; // this includes the zeroth segment
+
+ public DefaultColumnMultiPageZeroWriter(IColumnWriteMultiPageOp multiPageOp, int zerothSegmentMaxColumns,
+ int bufferCapacity) {
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef = new MutableObject<>();
+ multiPageOpRef.setValue(multiPageOp);
+ segments = new MultiPersistentPageZeroBufferBytesOutputStream(multiPageOpRef); // should this be populated at reset?
+ this.zerothSegmentWriter = new DefaultColumnPageZeroWriter();
+ this.zerothSegmentMaxColumns = zerothSegmentMaxColumns;
+ this.maximumNumberOfColumnsInAPage = getMaximumNumberOfColumnsInAPage(bufferCapacity);
+ }
+
+ @Override
+ public void resetBasedOnColumns(int[] presentColumns, int numberOfColumns) throws HyracksDataException {
+ this.numberOfColumns = numberOfColumns;
+ this.numberOfColumnInZerothSegment = Math.min(numberOfColumns, zerothSegmentMaxColumns);
+ this.numberOfPageZeroSegments = calculateNumberOfPageZeroSegments(numberOfColumns,
+ numberOfColumnInZerothSegment, maximumNumberOfColumnsInAPage);
+ zerothSegmentWriter.resetBasedOnColumns(presentColumns, numberOfColumnInZerothSegment, EXTENDED_HEADER_SIZE);
+ if (numberOfPageZeroSegments > 1) {
+ // these many buffers need to be allocated, to get contiguous pageIds
+ segments.reset(numberOfPageZeroSegments - 1);
+ }
+ }
+
+ @Override
+ public void resetBasedOnColumns(int[] presentColumns, int numberOfColumns, int headerSize)
+ throws HyracksDataException {
+ throw new UnsupportedOperationException(
+ "resetBasedOnColumns with headerSize is not supported in multi-page zero writer");
+ }
+
+ private int calculateNumberOfPageZeroSegments(int numberOfColumns, int numberOfColumnInZerothSegment,
+ int maximumNumberOfColumnsInAPage) {
+ // calculate the number of segments required to store the columns
+ int numberOfColumnsBeyondZerothSegment = numberOfColumns - numberOfColumnInZerothSegment;
+ if (numberOfColumnsBeyondZerothSegment <= 0) {
+ return 1; // only zeroth segment is needed
+ }
+ return 1 + (int) Math.ceil((double) numberOfColumnsBeyondZerothSegment / maximumNumberOfColumnsInAPage);
+ }
+
+ @Override
+ public void allocateColumns() {
+ // allocate the zeroth segment columns
+ zerothSegmentWriter.allocateColumns();
+ // rest of the segments need not need to be allocated
+ // as those are full of columns
+ }
+
+ @Override
+ public void putColumnOffset(int columnIndex, int relativeColumnIndex, int offset) throws HyracksDataException {
+ try {
+ // for default writer, both columnIndex and relativeColumnIndex are the same
+ if (columnIndex < zerothSegmentMaxColumns) {
+ zerothSegmentWriter.putColumnOffset(columnIndex, relativeColumnIndex, offset);
+ } else {
+ // For columns beyond the zeroth segment, we need to write to the segments
+ int columnIndexInSegment = columnIndex - numberOfColumnInZerothSegment;
+ int requiredSegment = columnIndexInSegment / maximumNumberOfColumnsInAPage;
+ int columnIndexInRequiredSegment = columnIndexInSegment % maximumNumberOfColumnsInAPage;
+ int offsetInSegment = columnIndexInRequiredSegment * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ segments.writeInSegment(requiredSegment, offsetInSegment, offset);
+ }
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public void putColumnFilter(int columnIndex, long normalizedMinValue, long normalizedMaxValue)
+ throws HyracksDataException {
+ try {
+ if (columnIndex < zerothSegmentMaxColumns) {
+ zerothSegmentWriter.putColumnFilter(columnIndex, normalizedMinValue, normalizedMaxValue);
+ } else {
+ // For columns beyond the zeroth segment, we need to write to the segments
+ int columnIndexInSegment = columnIndex - numberOfColumnInZerothSegment;
+ int requiredSegment = columnIndexInSegment / maximumNumberOfColumnsInAPage;
+ int columnIndexInRequiredSegment = columnIndexInSegment % maximumNumberOfColumnsInAPage;
+ int numberOfColumnsInSegment = findNumberOfColumnsInSegment(requiredSegment);
+ int segmentFilterOffset = numberOfColumnsInSegment * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ int offsetInSegment =
+ segmentFilterOffset + columnIndexInRequiredSegment * DefaultColumnPageZeroWriter.FILTER_SIZE;
+ segments.writeInSegment(requiredSegment, offsetInSegment, normalizedMinValue);
+ segments.writeInSegment(requiredSegment, offsetInSegment + Long.BYTES, normalizedMaxValue);
+ }
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private int findNumberOfColumnsInSegment(int segmentIndex) {
+ // starts from 1st segment, not from 0th segment
+ if (segmentIndex == numberOfPageZeroSegments - 2) {
+ return numberOfColumns - numberOfColumnInZerothSegment
+ - (numberOfPageZeroSegments - 2) * maximumNumberOfColumnsInAPage;
+ }
+ // For segments beyond the zeroth segment, we can have maximum number of columns in a page, except the last segment.
+ return maximumNumberOfColumnsInAPage;
+ }
+
+ @Override
+ public void writePrimaryKeyColumns(IValuesWriter[] primaryKeyWriters) throws HyracksDataException {
+ // primary key columns are always written to the zeroth segment
+ zerothSegmentWriter.writePrimaryKeyColumns(primaryKeyWriters);
+ }
+
+ @Override
+ public byte flagCode() {
+ return MULTI_PAGE_DEFAULT_WRITER_FLAG;
+ }
+
+ @Override
+ public void flush(ByteBuffer buf, int numberOfTuples, ITupleReference minKey, ITupleReference maxKey,
+ AbstractColumnTupleWriter columnWriter, ITreeIndexTupleWriter rowTupleWriter) throws HyracksDataException {
+ buf.position(EXTENDED_HEADER_SIZE);
+ zerothSegmentWriter.setPageZero(buf);
+ buf.putInt(MEGA_LEAF_NODE_LENGTH, columnWriter.flush(this));
+ // Write min and max keys
+ int offset = buf.position();
+ buf.putInt(LEFT_MOST_KEY_OFFSET, offset);
+ offset += rowTupleWriter.writeTuple(minKey, buf.array(), offset);
+ buf.putInt(RIGHT_MOST_KEY_OFFSET, offset);
+ rowTupleWriter.writeTuple(maxKey, buf.array(), offset);
+
+ // Write page information
+ buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples);
+ buf.put(FLAG_OFFSET, flagCode());
+ buf.putInt(NUMBER_OF_COLUMNS_OFFSET, getNumberOfColumns());
+ buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, getColumnOffsetsSize());
+ // write the number of segments
+ buf.putInt(NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET, numberOfPageZeroSegments);
+ // write the number of columns in the zeroth segment
+ buf.putInt(MAX_COLUMNS_IN_ZEROTH_SEGMENT, zerothSegmentMaxColumns);
+
+ // reset the collected meta info
+ segments.finish();
+ columnWriter.reset();
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return numberOfColumns;
+ }
+
+ @Override
+ public boolean includeOrderedColumn(BitSet presentColumns, int columnIndex, boolean includeChildrenColumns) {
+ return true;
+ }
+
+ @Override
+ public int getPageZeroBufferCapacity() {
+ int pageSize = zerothSegmentWriter.getPageZeroBufferCapacity();
+ return pageSize * numberOfPageZeroSegments;
+ }
+
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) {
+ return columnIndex;
+ }
+
+ @Override
+ public int getColumnOffsetsSize() {
+ return numberOfColumns * DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+
+ @Override
+ public void setPageZero(ByteBuffer pageZero) {
+ throw new IllegalStateException("setPageZero is not supported in multi-page zero writer");
+ }
+
+ @Override
+ public int getHeaderSize() {
+ return EXTENDED_HEADER_SIZE;
+ }
+
+ public static int getMaximumNumberOfColumnsInAPage(int bufferCapacity) {
+ return bufferCapacity
+ / (DefaultColumnPageZeroWriter.COLUMN_OFFSET_SIZE + DefaultColumnPageZeroWriter.FILTER_SIZE);
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
new file mode 100644
index 0000000..035db5d
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.writers.multipage;
+
+import static org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter.MAX_COLUMNS_IN_ZEROTH_SEGMENT;
+import static org.apache.asterix.column.zero.writers.multipage.DefaultColumnMultiPageZeroWriter.NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET;
+import static org.apache.asterix.column.zero.writers.multipage.SparseColumnMultiPageZeroWriter.MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NEXT_LEAF_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NUMBER_OF_COLUMNS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.RIGHT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.TUPLE_COUNT_OFFSET;
+
+import java.io.EOFException;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.asterix.column.zero.readers.SparseColumnPageZeroReader;
+import org.apache.asterix.column.zero.writers.SparseColumnPageZeroWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+
+public class SparseColumnMultiPageZeroReader extends AbstractColumnMultiPageZeroReader {
+ private final SparseColumnPageZeroReader zerothSegmentReader;
+ private final int maxNumberOfColumnsInAPage;
+ private final BitSet pageZeroSegmentsPages;
+ private final Int2IntOpenHashMap columnIndexToRelativeColumnIndex;
+
+ private int maxColumnIndexInZerothSegment;
+ private int numberOfColumnInZerothSegment;
+ private int numberOfPageZeroSegments;
+ private int headerSize;
+ private ByteBuffer pageZeroBuf;
+
+ private final VoidPointable offsetPointable;
+
+ public SparseColumnMultiPageZeroReader(int bufferCapacity) {
+ super();
+ zerothSegmentReader = new SparseColumnPageZeroReader();
+ this.pageZeroSegmentsPages = new BitSet();
+ this.maxNumberOfColumnsInAPage =
+ SparseColumnMultiPageZeroWriter.getMaximumNumberOfColumnsInAPage(bufferCapacity);
+ this.offsetPointable = new VoidPointable();
+ this.columnIndexToRelativeColumnIndex = new Int2IntOpenHashMap();
+ columnIndexToRelativeColumnIndex.defaultReturnValue(-1);
+ }
+
+ @Override
+ public void resetStream(IColumnBufferProvider pageZeroSegmentBufferProvider) throws HyracksDataException {
+ segmentBuffers.reset(pageZeroSegmentBufferProvider);
+ }
+
+ @Override
+ public void reset(ByteBuffer pageZeroBuf) {
+ this.pageZeroBuf = pageZeroBuf;
+ numberOfPageZeroSegments = pageZeroBuf.getInt(NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET);
+ numberOfColumnInZerothSegment = pageZeroBuf.getInt(MAX_COLUMNS_IN_ZEROTH_SEGMENT);
+ maxColumnIndexInZerothSegment = pageZeroBuf.getInt(MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET);
+ headerSize = MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET + numberOfPageZeroSegments * Integer.BYTES;
+ zerothSegmentReader.reset(pageZeroBuf, Math.min(numberOfColumnInZerothSegment, getNumberOfPresentColumns()),
+ headerSize);
+ columnIndexToRelativeColumnIndex.clear();
+ }
+
+ @Override
+ public void reset(ByteBuffer pageZeroBuf, int headerSize) {
+ throw new UnsupportedOperationException("This method is not supported for multi-page zero readers.");
+ }
+
+ @Override
+ public int getColumnOffset(int columnIndex) throws HyracksDataException {
+ try {
+ if (columnIndex <= maxColumnIndexInZerothSegment) {
+ return zerothSegmentReader.getColumnOffset(columnIndex);
+ } else {
+ int segmentIndex = findSegment(columnIndex) - 1;
+ int relativeColumnIndex = findRelativeColumnIndex(columnIndex);
+ int columnIndexInRequiredSegment =
+ (relativeColumnIndex - numberOfColumnInZerothSegment) % maxNumberOfColumnsInAPage;
+ int segmentOffset =
+ columnIndexInRequiredSegment * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE + Integer.BYTES; // skipping 4 bytes of columnIndex
+ segmentBuffers.read(segmentIndex, offsetPointable, segmentOffset,
+ SparseColumnPageZeroWriter.COLUMN_INDEX_SIZE);
+ return IntegerPointable.getInteger(offsetPointable.getByteArray(), offsetPointable.getStartOffset());
+ }
+ } catch (EOFException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private int findSegment(int columnIndex) {
+ // This method finds the segment index (except for 0th segment) for the given columnIndex.
+ if (numberOfPageZeroSegments == 1) {
+ // only zeroth segment is present
+ return 0;
+ }
+ // gives 0 based segment index (0 for zeroth segment, 1 for first segment, etc.)
+ int start = 1;
+ int end = numberOfPageZeroSegments - 1;
+ int resultSegment = -1;
+ while (start <= end) {
+ int mid = (start + end) / 2;
+ int segmentColumnIndex =
+ pageZeroBuf.getInt(MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET + mid * Integer.BYTES);
+ if (segmentColumnIndex >= columnIndex) {
+ resultSegment = mid;
+ end = mid - 1; // continue searching in the left half
+ } else {
+ start = mid + 1;
+ }
+ }
+ return resultSegment;
+ }
+
+ private int findRelativeColumnIndex(int columnIndex) throws HyracksDataException {
+ if (columnIndexToRelativeColumnIndex.get(columnIndex) != -1) {
+ return columnIndexToRelativeColumnIndex.get(columnIndex);
+ }
+ if (columnIndex <= maxColumnIndexInZerothSegment) {
+ return zerothSegmentReader.getRelativeColumnIndex(columnIndex);
+ } else {
+ int segmentIndex = findSegment(columnIndex);
+ if (segmentIndex <= 0) {
+ return -1;
+ }
+ segmentIndex -= 1; // Adjusting to get the segment index for the segment stream
+ // Oth based segment index, hence need to check in segmentIndex - 1 th buffer
+ int numberOfColumnsInSegment =
+ segmentIndex == numberOfPageZeroSegments - 2
+ ? getNumberOfPresentColumns() - numberOfColumnInZerothSegment
+ - (numberOfPageZeroSegments - 2) * maxNumberOfColumnsInAPage
+ : maxNumberOfColumnsInAPage;
+ int segmentColumnIndex =
+ segmentBuffers.findColumnIndexInSegment(segmentIndex, columnIndex, numberOfColumnsInSegment);
+ if (segmentColumnIndex == -1) {
+ return -1;
+ }
+ int relativeIndex =
+ numberOfColumnInZerothSegment + segmentIndex * maxNumberOfColumnsInAPage + segmentColumnIndex;
+ columnIndexToRelativeColumnIndex.put(columnIndex, relativeIndex);
+ return relativeIndex;
+ }
+ }
+
+ private int findNumberOfColumnsInSegment(int segmentIndex) {
+ // starts from 1st segment, not from 0th segment
+ if (segmentIndex == numberOfPageZeroSegments - 2) {
+ return getNumberOfPresentColumns() - numberOfColumnInZerothSegment
+ - (numberOfPageZeroSegments - 2) * maxNumberOfColumnsInAPage;
+ }
+ // For segments beyond the zeroth segment, we can have maximum number of columns in a page, except the last segment.
+ return maxNumberOfColumnsInAPage;
+ }
+
+ @Override
+ public long getColumnFilterMin(int columnIndex) throws HyracksDataException {
+ try {
+ if (columnIndex <= maxColumnIndexInZerothSegment) {
+ return zerothSegmentReader.getColumnFilterMin(columnIndex);
+ } else {
+ int segmentIndex = findSegment(columnIndex) - 1;
+ int relativeColumnIndex = findRelativeColumnIndex(columnIndex);
+ int columnIndexInRequiredSegment =
+ (relativeColumnIndex - numberOfColumnInZerothSegment) % maxNumberOfColumnsInAPage;
+ int segmentOffset =
+ findNumberOfColumnsInSegment(segmentIndex) * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ segmentOffset += columnIndexInRequiredSegment * SparseColumnPageZeroWriter.FILTER_SIZE;
+ segmentBuffers.read(segmentIndex, offsetPointable, segmentOffset, Long.BYTES);
+ return LongPointable.getLong(offsetPointable.getByteArray(), offsetPointable.getStartOffset());
+ }
+ } catch (EOFException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public long getColumnFilterMax(int columnIndex) throws HyracksDataException {
+ try {
+ if (columnIndex <= maxColumnIndexInZerothSegment) {
+ return zerothSegmentReader.getColumnFilterMax(columnIndex);
+ } else {
+ int segmentIndex = findSegment(columnIndex) - 1;
+ int relativeColumnIndex = findRelativeColumnIndex(columnIndex);
+ int columnIndexInRequiredSegment =
+ (relativeColumnIndex - numberOfColumnInZerothSegment) % maxNumberOfColumnsInAPage;
+ int segmentOffset =
+ findNumberOfColumnsInSegment(segmentIndex) * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ segmentOffset += columnIndexInRequiredSegment * SparseColumnPageZeroWriter.FILTER_SIZE;
+ segmentOffset += Long.BYTES; // skip min filter
+ segmentBuffers.read(segmentIndex, offsetPointable, segmentOffset, Long.BYTES);
+ return LongPointable.getLong(offsetPointable.getByteArray(), offsetPointable.getStartOffset());
+ }
+ } catch (EOFException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public void skipFilters() {
+ zerothSegmentReader.skipFilters();
+ }
+
+ @Override
+ public void skipColumnOffsets() {
+ zerothSegmentReader.skipColumnOffsets();
+ }
+
+ @Override
+ public int getTupleCount() {
+ return pageZeroBuf.getInt(TUPLE_COUNT_OFFSET);
+ }
+
+ @Override
+ public int getLeftMostKeyOffset() {
+ return pageZeroBuf.getInt(LEFT_MOST_KEY_OFFSET);
+ }
+
+ @Override
+ public int getRightMostKeyOffset() {
+ return pageZeroBuf.getInt(RIGHT_MOST_KEY_OFFSET);
+ }
+
+ @Override
+ public int getNumberOfPresentColumns() {
+ return pageZeroBuf.getInt(NUMBER_OF_COLUMNS_OFFSET);
+ }
+
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) throws HyracksDataException {
+ return findRelativeColumnIndex(columnIndex);
+ }
+
+ @Override
+ public int getNextLeaf() {
+ return pageZeroBuf.getInt(NEXT_LEAF_OFFSET);
+ }
+
+ @Override
+ public int getMegaLeafNodeLengthInBytes() {
+ return pageZeroBuf.getInt(MEGA_LEAF_NODE_LENGTH);
+ }
+
+ @Override
+ public int getPageZeroCapacity() {
+ return pageZeroBuf.capacity();
+ }
+
+ @Override
+ public boolean isValidColumn(int columnIndex) throws HyracksDataException {
+ return findRelativeColumnIndex(columnIndex) != -1;
+ }
+
+ @Override
+ public void getAllColumns(BitSet presentColumns) {
+ int columnOffsetStart = headerSize;
+ for (int i = 0; i < Math.min(getNumberOfPresentColumns(), numberOfColumnInZerothSegment); i++) {
+ int columnIndex = pageZeroBuf.getInt(columnOffsetStart);
+ presentColumns.set(columnIndex);
+ columnOffsetStart += SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+ if (getNumberOfPresentColumns() > numberOfColumnInZerothSegment) {
+ // read the rest of the columns from the segment stream
+ int columnsInLastSegment = getNumberOfPresentColumns() - numberOfColumnInZerothSegment
+ - (numberOfPageZeroSegments - 2) * maxNumberOfColumnsInAPage;
+ segmentBuffers.readAllColumns(presentColumns, numberOfPageZeroSegments, maxNumberOfColumnsInAPage,
+ columnsInLastSegment);
+ }
+ }
+
+ @Override
+ public ByteBuffer getPageZeroBuf() {
+ throw new UnsupportedOperationException("This method is not supported for multi-page zero readers.");
+ }
+
+ @Override
+ public int populateOffsetColumnIndexPairs(long[] offsetColumnIndexPairs) {
+ // offsetColumnIndexPairs >= getNumberOfPresentColumns() + 1 (maybe because of the previous MegaLeaf).
+ // Do not rely on offsetColumnIndexPairs.length, as it may be larger than the number of present columns.
+ // This is because the same array is reused for multiple leaf segments, and previous leaves may have more columns.
+ int columnOffsetStart = headerSize;
+ int currentColumnIndex = 0;
+ int numberOfColumns = getNumberOfPresentColumns();
+ while (currentColumnIndex < Math.min(numberOfColumns, numberOfColumnInZerothSegment)) {
+ int columnIndex = pageZeroBuf.getInt(columnOffsetStart);
+ int columnOffset = pageZeroBuf.getInt(columnOffsetStart + SparseColumnPageZeroWriter.COLUMN_INDEX_SIZE);
+ offsetColumnIndexPairs[currentColumnIndex++] = IntPairUtil.of(columnOffset, columnIndex);
+ columnOffsetStart += SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+
+ // If the pages are not pinned, we will not read any columnIndex, but the old stuffs will already be present in the offsetColumnIndexPairs.
+ if (numberOfColumns > numberOfColumnInZerothSegment) {
+ // read the rest of the columns from the segment stream
+ int columnsInLastSegment = getNumberOfPresentColumns() - numberOfColumnInZerothSegment
+ - (numberOfPageZeroSegments - 2) * maxNumberOfColumnsInAPage;
+ currentColumnIndex = segmentBuffers.readSparseOffset(offsetColumnIndexPairs, numberOfPageZeroSegments,
+ maxNumberOfColumnsInAPage, columnsInLastSegment, currentColumnIndex);
+ }
+
+ return currentColumnIndex;
+ }
+
+ @Override
+ public int getNumberOfPageZeroSegments() {
+ return numberOfPageZeroSegments;
+ }
+
+ @Override
+ public BitSet getPageZeroSegmentsPages() {
+ return pageZeroSegmentsPages;
+ }
+
+ @Override
+ public int getHeaderSize() {
+ return headerSize;
+ }
+
+ @Override
+ public BitSet markRequiredPageSegments(BitSet projectedColumns, int pageZeroId, boolean markAll) {
+ pageZeroSegmentsPages.clear();
+ // Not marking the zeroth segment
+ if (numberOfPageZeroSegments == 1 || markAll) {
+ // mark all segments as required
+ pageZeroSegmentsPages.set(1, numberOfPageZeroSegments);
+ } else {
+ // Iterate over the projected columns and mark the segments that contain them
+ int currentIndex = projectedColumns.nextSetBit(maxColumnIndexInZerothSegment + 1);
+ while (currentIndex >= 0) {
+ int rangeEnd = projectedColumns.nextClearBit(currentIndex); // exclusive
+ int startSegmentIndex = findSegment(currentIndex);
+ if (startSegmentIndex == -1) {
+ //This indicates that the currentIndex > MaxColumnIndex in the last segment
+ //Hence this leaf doesn't need to pin the segment for requested column ranges.
+
+ //We can return early as next projectedColumns next set bit will also be out of bounds.
+ break;
+ }
+ int endSegmentIndex = findSegment(rangeEnd - 1);
+ if (endSegmentIndex == -1) {
+ //This indicates that the rangeEnd - 1 > MaxColumnIndex in the last segment
+ //but the startSegmentIndex is valid, hence we may pin to the last segment.
+ endSegmentIndex = numberOfPageZeroSegments - 1; // Last segment index
+ }
+
+ if (startSegmentIndex <= endSegmentIndex) {
+ pageZeroSegmentsPages.set(startSegmentIndex, endSegmentIndex + 1);
+ }
+
+ currentIndex = projectedColumns.nextSetBit(rangeEnd);
+ }
+ }
+ return pageZeroSegmentsPages;
+ }
+
+ @Override
+ public void unPinNotRequiredPageZeroSegments() throws HyracksDataException {
+ segmentBuffers.unPinNotRequiredSegments(pageZeroSegmentsPages, numberOfPageZeroSegments);
+ }
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("maxColumnIndexInZerothSegment", maxColumnIndexInZerothSegment);
+ readerNode.put("numberOfColumnInZerothSegment", numberOfColumnInZerothSegment);
+ readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage);
+ readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments);
+ LOGGER.debug("SparseColumnMultiPageZeroReader Info: {}", readerNode.toPrettyString());
+ }
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroWriter.java
new file mode 100644
index 0000000..5753632
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroWriter.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.zero.writers.multipage;
+
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.FLAG_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.HEADER_SIZE;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NUMBER_OF_COLUMNS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.RIGHT_MOST_KEY_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.SIZE_OF_COLUMNS_OFFSETS_OFFSET;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.TUPLE_COUNT_OFFSET;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.asterix.column.bytes.stream.out.MultiPersistentPageZeroBufferBytesOutputStream;
+import org.apache.asterix.column.zero.writers.SparseColumnPageZeroWriter;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IValuesWriter;
+
+/*
+[ PageZero Segment 0 ]
+──────────────────────────────────────────────────────────────────────────────
+| Headers |
+| ───────────────────────────────────────────────────────────────────────── |
+| TupleCountOffset |
+| MaxColumnsInZerothSegment |
+| LevelOffset |
+| NumberOfColumnsOffset |
+| LeftMostKeyOffset |
+| RightMostKeyOffset |
+| SizeOfColumnsOffsetsOffset |
+| MegaLeafNodeLength |
+| FlagOffset |
+| NextLeafOffset |
+| NumberOfPageSegments |
+| MaxColumnIndexInZerothSegment |
+| MaxColumnIndexInFirstSegment |
+| MaxColumnIndexInThirdSegment |
+
+| Min Primary Key |
+| Max Primary Key |
+| Primary Key Values |
+| [ offset₁, min₁, max₁ ] |
+| [ offset₂, min₂, max₂ ] |
+| [ offset₃, min₃, max₃ ] |
+| ... |
+
+[ PageZero Segment 1..N ]
+──────────────────────────────────────────────────────────────────────────────
+| Additional column metadata (same format) |
+*/
+public class SparseColumnMultiPageZeroWriter implements IColumnPageZeroWriter {
+ //For storing the last columnIndex in the ith segment
+ public static final int NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET = HEADER_SIZE;
+ public static final int MAX_COLUMNS_IN_ZEROTH_SEGMENT_OFFSET = NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET + 4;
+ public static final int MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET = MAX_COLUMNS_IN_ZEROTH_SEGMENT_OFFSET + 4;
+
+ private final MultiPersistentPageZeroBufferBytesOutputStream segments;
+ private final SparseColumnPageZeroWriter zerothSegmentWriter;
+ private final int maximumNumberOfColumnsInAPage;
+ private final int zerothSegmentMaxColumns;
+ private int[] presentColumns;
+ private int numberOfPresentColumns;
+ private int numberOfPageZeroSegments;
+ private int numberOfColumnInZerothSegment;
+
+ public SparseColumnMultiPageZeroWriter(IColumnWriteMultiPageOp multiPageOp, int zerothSegmentMaxColumns,
+ int bufferCachePageSize) {
+ Mutable<IColumnWriteMultiPageOp> multiPageOpRef = new MutableObject<>();
+ multiPageOpRef.setValue(multiPageOp);
+ segments = new MultiPersistentPageZeroBufferBytesOutputStream(multiPageOpRef);
+ this.zerothSegmentMaxColumns = zerothSegmentMaxColumns;
+ this.zerothSegmentWriter = new SparseColumnPageZeroWriter();
+ this.maximumNumberOfColumnsInAPage = getMaximumNumberOfColumnsInAPage(bufferCachePageSize);
+ }
+
+ @Override
+ public void resetBasedOnColumns(int[] presentColumns, int numberOfColumns) throws HyracksDataException {
+ this.presentColumns = presentColumns;
+ this.numberOfPresentColumns = presentColumns.length;
+ this.numberOfColumnInZerothSegment = Math.min(numberOfPresentColumns, zerothSegmentMaxColumns);
+ this.numberOfPageZeroSegments = calculateNumberOfPageZeroSegments(numberOfPresentColumns,
+ numberOfColumnInZerothSegment, maximumNumberOfColumnsInAPage);
+ int headerSize = MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET + numberOfPageZeroSegments * Integer.BYTES;
+ zerothSegmentWriter.resetInnerBasedOnColumns(presentColumns, numberOfColumnInZerothSegment, headerSize);
+ if (numberOfPageZeroSegments > 1) {
+ segments.reset(numberOfPageZeroSegments - 1);
+ }
+ }
+
+ @Override
+ public void resetBasedOnColumns(int[] presentColumns, int numberOfColumns, int headerSize)
+ throws HyracksDataException {
+ throw new UnsupportedOperationException(
+ "resetBasedOnColumns with headerSize is not supported in multi-page zero writer");
+ }
+
+ @Override
+ public byte flagCode() {
+ return MULTI_PAGE_SPARSE_WRITER_FLAG;
+ }
+
+ private int calculateNumberOfPageZeroSegments(int numberOfColumns, int numberOfColumnInZerothSegment,
+ int maximumNumberOfColumnsInAPage) {
+ // calculate the number of segments required to store the columns
+ int numberOfColumnsBeyondZerothSegment = numberOfColumns - numberOfColumnInZerothSegment;
+ if (numberOfColumnsBeyondZerothSegment <= 0) {
+ return 1; // only zeroth segment is needed
+ }
+ return 1 + (int) Math.ceil((double) numberOfColumnsBeyondZerothSegment / maximumNumberOfColumnsInAPage);
+ }
+
+ @Override
+ public void allocateColumns() {
+ // allocate the zeroth segment columns
+ zerothSegmentWriter.allocateColumns();
+ }
+
+ @Override
+ public void putColumnOffset(int absoluteColumnIndex, int relativeColumnIndex, int offset)
+ throws HyracksDataException {
+ // for sparse writer, we need to find the relative column index in the present columns.
+ try {
+ if (relativeColumnIndex < zerothSegmentMaxColumns) {
+ // Write to the zeroth segment
+ zerothSegmentWriter.putColumnOffset(absoluteColumnIndex, relativeColumnIndex, offset);
+ } else {
+ int columnIndexInSegment = relativeColumnIndex - numberOfColumnInZerothSegment;
+ int requiredSegment = columnIndexInSegment / maximumNumberOfColumnsInAPage;
+ int columnIndexInRequiredSegment = columnIndexInSegment % maximumNumberOfColumnsInAPage;
+ int offsetInSegment = columnIndexInRequiredSegment * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ segments.writeInSegment(requiredSegment, offsetInSegment, absoluteColumnIndex, offset);
+ }
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public void putColumnFilter(int relativeColumnIndex, long normalizedMinValue, long normalizedMaxValue)
+ throws HyracksDataException {
+ try {
+ if (relativeColumnIndex < zerothSegmentMaxColumns) {
+ zerothSegmentWriter.putColumnFilter(relativeColumnIndex, normalizedMinValue, normalizedMaxValue);
+ } else {
+ // For columns beyond the zeroth segment, we need to write to the segments
+ int columnIndexInSegment = relativeColumnIndex - numberOfColumnInZerothSegment;
+ int requiredSegment = columnIndexInSegment / maximumNumberOfColumnsInAPage;
+ int columnIndexInRequiredSegment = columnIndexInSegment % maximumNumberOfColumnsInAPage;
+ int numberOfColumnsInSegment = findNumberOfColumnsInSegment(requiredSegment);
+ int segmentFilterOffset = numberOfColumnsInSegment * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ int offsetInSegment =
+ segmentFilterOffset + columnIndexInRequiredSegment * SparseColumnPageZeroWriter.FILTER_SIZE;
+ segments.writeInSegment(requiredSegment, offsetInSegment, normalizedMinValue);
+ segments.writeInSegment(requiredSegment, offsetInSegment + Long.BYTES, normalizedMaxValue);
+ }
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private int findNumberOfColumnsInSegment(int segmentIndex) {
+ // starts from 1st segment, not from 0th segment
+ if (segmentIndex == numberOfPageZeroSegments - 2) {
+ return numberOfPresentColumns - numberOfColumnInZerothSegment
+ - (numberOfPageZeroSegments - 2) * maximumNumberOfColumnsInAPage;
+ }
+ // For segments beyond the zeroth segment, we can have maximum number of columns in a page, except the last segment.
+ return maximumNumberOfColumnsInAPage;
+ }
+
+ @Override
+ public void writePrimaryKeyColumns(IValuesWriter[] primaryKeyWriters) throws HyracksDataException {
+ zerothSegmentWriter.writePrimaryKeyColumns(primaryKeyWriters);
+ }
+
+ @Override
+ public int getNumberOfColumns() {
+ return numberOfPresentColumns;
+ }
+
+ @Override
+ public boolean includeOrderedColumn(BitSet presentColumns, int columnIndex, boolean includeChildrenColumns) {
+ return zerothSegmentWriter.includeOrderedColumn(presentColumns, columnIndex, includeChildrenColumns);
+ }
+
+ @Override
+ public int getPageZeroBufferCapacity() {
+ int pageSize = zerothSegmentWriter.getPageZeroBufferCapacity();
+ return pageSize * numberOfPageZeroSegments;
+ }
+
+ @Override
+ public int getRelativeColumnIndex(int columnIndex) {
+ int relativeColumnIndex =
+ zerothSegmentWriter.findColumnIndex(presentColumns, numberOfPresentColumns, columnIndex);
+ if (relativeColumnIndex == -1) {
+ throw new IllegalStateException("Column index " + relativeColumnIndex + " is out of bounds");
+ }
+ return relativeColumnIndex;
+ }
+
+ @Override
+ public int getColumnOffsetsSize() {
+ return numberOfPresentColumns * SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE;
+ }
+
+ @Override
+ public void setPageZero(ByteBuffer pageZero) {
+ throw new IllegalStateException("setPageZero is not supported in multi-page zero writer");
+ }
+
+ @Override
+ public void flush(ByteBuffer buf, int numberOfTuples, ITupleReference minKey, ITupleReference maxKey,
+ AbstractColumnTupleWriter columnWriter, ITreeIndexTupleWriter rowTupleWriter) throws HyracksDataException {
+ zerothSegmentWriter.setPageZero(buf);
+ buf.putInt(MEGA_LEAF_NODE_LENGTH, columnWriter.flush(this));
+ // Write min and max keys
+ int offset = buf.position();
+ buf.putInt(LEFT_MOST_KEY_OFFSET, offset);
+ offset += rowTupleWriter.writeTuple(minKey, buf.array(), offset);
+ buf.putInt(RIGHT_MOST_KEY_OFFSET, offset);
+ rowTupleWriter.writeTuple(maxKey, buf.array(), offset);
+
+ // Write page information
+ buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples);
+ buf.put(FLAG_OFFSET, flagCode());
+ buf.putInt(NUMBER_OF_COLUMNS_OFFSET, getNumberOfColumns());
+ buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, getColumnOffsetsSize());
+ // write the number of segments
+ buf.putInt(NUMBER_OF_PAGE_ZERO_SEGMENTS_OFFSET, numberOfPageZeroSegments);
+ // write the max column count in the zeroth segment
+ buf.putInt(MAX_COLUMNS_IN_ZEROTH_SEGMENT_OFFSET, numberOfColumnInZerothSegment);
+
+ // write the max columnIndex in headers.
+ for (int i = 0; i < numberOfPageZeroSegments; i++) {
+ int columnIndexOffset = MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET + i * Integer.BYTES;
+ if (i == 0) {
+ int presentColumnIndex = numberOfColumnInZerothSegment - 1;
+ buf.putInt(columnIndexOffset, presentColumns[presentColumnIndex]);
+ } else if (i == numberOfPageZeroSegments - 1) {
+ buf.putInt(columnIndexOffset, presentColumns[numberOfPresentColumns - 1]);
+ } else {
+ int presentColumnIndex = numberOfColumnInZerothSegment + i * maximumNumberOfColumnsInAPage;
+ buf.putInt(columnIndexOffset, presentColumns[presentColumnIndex - 1]);
+ }
+ }
+
+ // reset the collected meta info
+ segments.finish();
+ columnWriter.reset();
+ }
+
+ @Override
+ public int getHeaderSize() {
+ return MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET + numberOfPageZeroSegments * Integer.BYTES;
+ }
+
+ public static int getHeaderSpace(int numberOfExtraPagesRequired) {
+ return MAX_COLUMNS_INDEX_IN_ZEROTH_SEGMENT_OFFSET + numberOfExtraPagesRequired * Integer.BYTES;
+ }
+
+ public static int getMaximumNumberOfColumnsInAPage(int bufferCachePageSize) {
+ return bufferCachePageSize
+ / (SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE + SparseColumnPageZeroWriter.FILTER_SIZE);
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/NoOpWriteMultiPageOp.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/NoOpWriteMultiPageOp.java
index 5c929c1..aa3cb71 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/NoOpWriteMultiPageOp.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/NoOpWriteMultiPageOp.java
@@ -35,6 +35,11 @@
}
@Override
+ public ByteBuffer confiscatePageZeroPersistent() throws HyracksDataException {
+ return null;
+ }
+
+ @Override
public ByteBuffer confiscateTemporary() throws HyracksDataException {
return null;
}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/TestWriteMultiPageOp.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/TestWriteMultiPageOp.java
index 8e01740..556a87c 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/TestWriteMultiPageOp.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/TestWriteMultiPageOp.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
public class TestWriteMultiPageOp implements IColumnWriteMultiPageOp {
@@ -37,6 +38,11 @@
}
@Override
+ public ByteBuffer confiscatePageZeroPersistent() throws HyracksDataException {
+ return dummyBufferCache.allocate(fileId).getBuffer();
+ }
+
+ @Override
public ByteBuffer confiscateTemporary() {
return dummyBufferCache.allocateTemporary();
}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java
index 9d57dde..1ef865e 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java
@@ -19,8 +19,6 @@
package org.apache.asterix.column.test.bytes;
import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.HEADER_SIZE;
-import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
-import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NUMBER_OF_COLUMNS_OFFSET;
import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.TUPLE_COUNT_OFFSET;
import java.io.File;
@@ -48,6 +46,7 @@
import org.apache.asterix.column.test.bytes.components.TestColumnBufferProvider;
import org.apache.asterix.column.values.IColumnValuesWriterFactory;
import org.apache.asterix.column.values.writer.ColumnValuesWriterFactory;
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
import org.apache.asterix.om.pointables.ARecordVisitablePointable;
import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.pointables.printer.json.clean.APrintVisitor;
@@ -63,6 +62,7 @@
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.write.DefaultColumnWriteContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
import org.apache.hyracks.util.StorageUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -189,14 +189,8 @@
protected void writeFullPage(ByteBuffer pageZero, AbstractColumnTupleWriter writer, int tupleCount)
throws HyracksDataException {
pageZero.clear();
- //Reserve the header space
- pageZero.position(HEADER_SIZE);
- pageZero.putInt(MEGA_LEAF_NODE_LENGTH, writer.flush(pageZero));
- //Write page header
- int numberOfColumn = writer.getNumberOfColumns(false);
- pageZero.putInt(TUPLE_COUNT_OFFSET, tupleCount);
- pageZero.putInt(NUMBER_OF_COLUMNS_OFFSET, numberOfColumn);
-
+ DefaultColumnPageZeroWriter pageZeroWriter = new DefaultColumnPageZeroWriter();
+ pageZeroWriter.flush(pageZero, tupleCount, writer);
}
protected boolean isFull(AbstractColumnTupleWriter columnWriter, int tupleCount, ITupleReference tuple) {
@@ -208,10 +202,13 @@
}
//Reserved for the number of pages
int requiredFreeSpace = HEADER_SIZE;
+ //Since this test uses DefaultWriter, it does not need the bufferCapacity in the calculation
+ int bufferCapacity = Integer.MAX_VALUE;
//Columns' Offsets
- requiredFreeSpace += columnWriter.getColumnOffsetsSize(true);
+ requiredFreeSpace += columnWriter.getPageZeroWriterOccupiedSpace(100, bufferCapacity, true,
+ IColumnPageZeroWriter.ColumnPageZeroWriterType.DEFAULT);
//Occupied space from previous writes
- requiredFreeSpace += columnWriter.getOccupiedSpace();
+ requiredFreeSpace += columnWriter.getPrimaryKeysEstimatedSize();
//New tuple required space
requiredFreeSpace += columnWriter.bytesRequired(tuple);
return PAGE_SIZE <= requiredFreeSpace;
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/AbstractDummyTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/AbstractDummyTest.java
index 1fafcce..9d6e619 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/AbstractDummyTest.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/AbstractDummyTest.java
@@ -19,6 +19,7 @@
package org.apache.asterix.column.test.dummy;
import java.io.IOException;
+import java.util.BitSet;
import java.util.Collections;
import org.apache.asterix.column.common.buffer.NoOpWriteMultiPageOp;
@@ -30,6 +31,7 @@
import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
import org.apache.asterix.column.values.writer.DummyColumnValuesWriterFactory;
import org.apache.asterix.column.values.writer.NoOpColumnBatchWriter;
+import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
import org.apache.asterix.om.lazy.RecordLazyVisitablePointable;
import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.commons.lang3.mutable.MutableObject;
@@ -41,6 +43,7 @@
protected final FlushColumnMetadata columnMetadata;
protected final ColumnTransformer columnTransformer;
protected final BatchFinalizerVisitor finalizer;
+ protected final BitSet presentColumnsIndexes;
//Schema
protected final ArrayBackedValueStorage storage;
protected final RecordLazyVisitablePointable pointable;
@@ -48,9 +51,10 @@
protected AbstractDummyTest(TestCase testCase) throws HyracksDataException {
super(testCase);
+ presentColumnsIndexes = new BitSet();
columnMetadata = new FlushColumnMetadata(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, null,
Collections.emptyList(), null, WRITER_FACTORY, new MutableObject<>(NoOpWriteMultiPageOp.INSTANCE));
- columnTransformer = new ColumnTransformer(columnMetadata, columnMetadata.getRoot());
+ columnTransformer = new ColumnTransformer(columnMetadata, columnMetadata.getRoot(), presentColumnsIndexes);
finalizer = new BatchFinalizerVisitor(columnMetadata);
storage = new ArrayBackedValueStorage();
pointable = new RecordLazyVisitablePointable(true);
@@ -64,7 +68,8 @@
storage.reset();
numberOfTuples++;
}
- finalizer.finalizeBatch(NoOpColumnBatchWriter.INSTANCE, columnMetadata);
+ finalizer.finalizeBatchColumns(columnMetadata, presentColumnsIndexes, new DefaultColumnPageZeroWriter());
+ finalizer.finalizeBatch(NoOpColumnBatchWriter.INSTANCE);
return columnMetadata.getRoot();
}
}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/values/writer/NoOpColumnBatchWriter.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/values/writer/NoOpColumnBatchWriter.java
index 234f804..0038fbe 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/values/writer/NoOpColumnBatchWriter.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/values/writer/NoOpColumnBatchWriter.java
@@ -18,12 +18,12 @@
*/
package org.apache.asterix.column.values.writer;
-import java.nio.ByteBuffer;
import java.util.PriorityQueue;
import org.apache.asterix.column.values.IColumnBatchWriter;
import org.apache.asterix.column.values.IColumnValuesWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
public class NoOpColumnBatchWriter implements IColumnBatchWriter {
public static final IColumnBatchWriter INSTANCE = new NoOpColumnBatchWriter();
@@ -32,7 +32,8 @@
}
@Override
- public void setPageZeroBuffer(ByteBuffer pageZeroBuffer, int numberOfColumns, int numberOfPrimaryKeys) {
+ public void setPageZeroWriter(IColumnPageZeroWriter pageZeroWriter, int[] presentColumnsIndexes,
+ int numberOfColumns) {
// NoOp
}
diff --git a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/sweeper/SweepContext.java b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/sweeper/SweepContext.java
index 973d9a1..86cac57 100644
--- a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/sweeper/SweepContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/sweeper/SweepContext.java
@@ -88,6 +88,10 @@
return cloudIOManager.punchHole(handle.getFileHandle(), offset, length);
}
+ public BufferCache getBufferCache() {
+ return bufferCache;
+ }
+
/**
* Whether the sweep operation should stop or proceed
* Stopping condition:
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 90bc499..e06d400 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -110,7 +110,9 @@
OptionTypes.STRING,
(Function<IApplicationConfig, String>) appConfig -> FileUtil
.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "passwd"),
- ControllerConfig.Option.DEFAULT_DIR.cmdline() + "/passwd");
+ ControllerConfig.Option.DEFAULT_DIR.cmdline() + "/passwd"),
+ STORAGE_MAX_COLUMNS_IN_ZEROTH_SEGMENT(INTEGER_BYTE_UNIT, 5000),
+ STORAGE_PAGE_ZERO_WRITER(STRING, "default");
private final IOptionType parser;
private final String defaultValueDescription;
@@ -267,6 +269,11 @@
return "Path to HTTP basic credentials";
case ABORT_TASKS_TIMEOUT:
return "The maximum time to wait for the tasks to be aborted";
+ case STORAGE_MAX_COLUMNS_IN_ZEROTH_SEGMENT:
+ return "The maximum number of columns in zero segment (default: 5000).";
+ case STORAGE_PAGE_ZERO_WRITER:
+ return "The config to choose between writers for page zero. (Possible values: default, sparse, adaptive), "
+ + "(default value: default)";
default:
throw new IllegalStateException("Not yet implemented: " + this);
}
@@ -646,6 +653,14 @@
return appConfig.getInt(Option.ABORT_TASKS_TIMEOUT);
}
+ public int getStorageMaxColumnsInZerothSegment() {
+ return appConfig.getInt(Option.STORAGE_MAX_COLUMNS_IN_ZEROTH_SEGMENT);
+ }
+
+ public String getStoragePageZeroWriter() {
+ return appConfig.getString(Option.STORAGE_PAGE_ZERO_WRITER);
+ }
+
public long getLibraryMaxFileSize() {
return appConfig.getLong(Option.LIBRARY_MAX_FILE_SIZE);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/pom.xml
index bbd8d87..72eae65 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/pom.xml
@@ -94,6 +94,11 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-cloud</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReader.java
index 7db792b..433d00e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleReader.java
@@ -18,33 +18,29 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.column.api;
-import java.nio.ByteBuffer;
-
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriterFlavorSelector;
/**
* Provided for columnar read tuple reference
*/
public abstract class AbstractColumnTupleReader extends AbstractTupleWriterDisabledMethods {
+ protected final IColumnPageZeroWriterFlavorSelector pageZeroWriterFlavorSelector;
+
+ protected AbstractColumnTupleReader(IColumnPageZeroWriterFlavorSelector pageZeroWriterFlavorSelector) {
+ this.pageZeroWriterFlavorSelector = pageZeroWriterFlavorSelector;
+ }
+
public abstract IColumnTupleIterator createTupleIterator(ColumnBTreeReadLeafFrame frame, int componentIndex,
IColumnReadMultiPageOp multiPageOp);
- /**
- * Currently fixed to 4-byte per offset
- *
- * @param buf buffer of Page0
- * @param columnIndex column index
- * @return column offset
- * @see AbstractColumnTupleWriter#getColumnOffsetsSize()
- */
- public final int getColumnOffset(ByteBuffer buf, int columnIndex) {
- return buf.getInt(AbstractColumnBTreeLeafFrame.HEADER_SIZE + columnIndex * Integer.BYTES);
- }
-
@Override
public final int bytesRequired(ITupleReference tuple) {
throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
}
+
+ public IColumnPageZeroWriterFlavorSelector getPageZeroWriterFlavorSelector() {
+ return pageZeroWriterFlavorSelector;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java
index 14f2399..8b3abd6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java
@@ -22,6 +22,8 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroWriterFlavorSelector;
/**
* Columnar Tuple Writer:
@@ -35,7 +37,7 @@
* - Initially, the writer has to set multiPageOp by calling {@link #init(IColumnWriteMultiPageOp)}
* - For each write, the caller should check if adding a tuple does not exceed the {@link #getMaxNumberOfTuples()} or
* the on-disk page size (called stopping condition)
- * - If the stopping condition is reached, then {@link #flush(ByteBuffer)} needed to be called
+ * - If the stopping condition is reached, then {@link #flush(ByteBuffer, IColumnPageZeroWriter)} needed to be called
* <p>
* Hyracks visibility:
* - Columns are written as blobs (i.e., not interpretable by Hyracks)
@@ -54,7 +56,7 @@
/**
* @return The current number of columns including the current tuple
*/
- public abstract int getNumberOfColumns(boolean includeCurrentTupleColumns);
+ public abstract int getAbsoluteNumberOfColumns(boolean includeCurrentTupleColumns);
/**
* Currently, a column offset takes 4-byte (fixed). But in the future, we can reformat the offsets. For example,
@@ -62,9 +64,8 @@
*
* @return the size needed to store columns' offsets
*/
- public final int getColumnOffsetsSize(boolean includeCurrentTupleColumns) {
- return Integer.BYTES * getNumberOfColumns(includeCurrentTupleColumns);
- }
+ public abstract int getPageZeroWriterOccupiedSpace(int maxColumnsInPageZerothSegment, int bufferCapacity,
+ boolean includeCurrentTupleColumns, IColumnPageZeroWriter.ColumnPageZeroWriterType adaptive);
/**
* @return maximum number of tuples to be stored per page (i.e., page0)
@@ -74,7 +75,7 @@
/**
* @return page0 occupied space
*/
- public abstract int getOccupiedSpace();
+ public abstract int getPrimaryKeysEstimatedSize();
/**
* Writes the tuple into a temporary internal buffers
@@ -88,10 +89,27 @@
*
* @return total flushed length (including page zero)
*/
- public abstract int flush(ByteBuffer pageZero) throws HyracksDataException;
+ public abstract int flush(IColumnPageZeroWriter columnPageZeroWriter) throws HyracksDataException;
/**
* Close the current writer and release all allocated temporary buffers
*/
public abstract void close();
+
+ /**
+ * reset the state after flush
+ */
+ public abstract void reset();
+
+ /**
+ * get the pageZero writer selector
+ * @return
+ */
+ public abstract IColumnPageZeroWriterFlavorSelector getColumnPageZeroWriterFlavorSelector();
+
+ public void setWriterType(IColumnPageZeroWriter.ColumnPageZeroWriterType pageZeroWriterType) {
+ if (pageZeroWriterType != IColumnPageZeroWriter.ColumnPageZeroWriterType.ADAPTIVE) {
+ getColumnPageZeroWriterFlavorSelector().setPageZeroWriterFlag(pageZeroWriterType.getWriterFlag());
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnWriteMultiPageOp.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnWriteMultiPageOp.java
index 2309fe1..b46b67d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnWriteMultiPageOp.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnWriteMultiPageOp.java
@@ -38,6 +38,8 @@
*/
ByteBuffer confiscatePersistent() throws HyracksDataException;
+ ByteBuffer confiscatePageZeroPersistent() throws HyracksDataException;
+
/**
* Persist all confiscated persistent buffers to disk
*/
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java
index eacf4fd..87b35bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java
@@ -25,10 +25,13 @@
import java.util.BitSet;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.read.CloudColumnReadContext;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.sweep.ColumnSweepPlanner;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.sweep.ColumnSweeper;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import it.unimi.dsi.fastutil.ints.IntArrays;
import it.unimi.dsi.fastutil.longs.LongArrays;
@@ -38,6 +41,7 @@
* Computes columns offsets, lengths, and pages
*/
public final class ColumnRanges {
+ private static final Logger LOGGER = LogManager.getLogger();
private static final LongComparator OFFSET_COMPARATOR = IntPairUtil.FIRST_COMPARATOR;
private final int numberOfPrimaryKeys;
@@ -79,7 +83,7 @@
*
* @param leafFrame to compute the ranges for
*/
- public void reset(ColumnBTreeReadLeafFrame leafFrame) {
+ public void reset(ColumnBTreeReadLeafFrame leafFrame) throws HyracksDataException {
reset(leafFrame, EMPTY, EMPTY, EMPTY);
}
@@ -89,7 +93,7 @@
* @param leafFrame to compute the ranges for
* @param plan eviction plan
*/
- public void reset(ColumnBTreeReadLeafFrame leafFrame, BitSet plan) {
+ public void reset(ColumnBTreeReadLeafFrame leafFrame, BitSet plan) throws HyracksDataException {
reset(leafFrame, plan, EMPTY, EMPTY);
}
@@ -102,62 +106,70 @@
* @param cloudOnlyColumns locked columns that cannot be read from a local disk
*/
public void reset(ColumnBTreeReadLeafFrame leafFrame, BitSet requestedColumns, BitSet evictableColumns,
- BitSet cloudOnlyColumns) {
- // Set leafFrame
- this.leafFrame = leafFrame;
- // Ensure arrays capacities (given the leafFrame's columns and pages)
- init();
+ BitSet cloudOnlyColumns) throws HyracksDataException {
+ try {
+ // Set leafFrame
+ this.leafFrame = leafFrame;
+ // Ensure arrays capacities (given the leafFrame's columns and pages)
+ init();
- // Get the number of columns in a page
- int numberOfColumns = leafFrame.getNumberOfColumns();
- for (int i = 0; i < numberOfColumns; i++) {
- int offset = leafFrame.getColumnOffset(i);
// Set the first 32-bits to the offset and the second 32-bits to columnIndex
- offsetColumnIndexPairs[i] = IntPairUtil.of(offset, i);
- }
+ int numberOfPresentColumnsInLeaf = leafFrame.populateOffsetColumnIndexPairs(offsetColumnIndexPairs);
- // Set artificial offset to determine the last column's length
- int megaLeafLength = leafFrame.getMegaLeafNodeLengthInBytes();
- offsetColumnIndexPairs[numberOfColumns] = IntPairUtil.of(megaLeafLength, numberOfColumns);
+ // Set artificial offset to determine the last column's length
+ int megaLeafLength = leafFrame.getMegaLeafNodeLengthInBytes();
+ offsetColumnIndexPairs[numberOfPresentColumnsInLeaf] =
+ IntPairUtil.of(megaLeafLength, numberOfPresentColumnsInLeaf);
- // Sort the pairs by offset (i.e., lowest offset first)
- LongArrays.stableSort(offsetColumnIndexPairs, 0, numberOfColumns, OFFSET_COMPARATOR);
+ // Sort the pairs by offset (i.e., lowest offset first)
+ LongArrays.stableSort(offsetColumnIndexPairs, 0, numberOfPresentColumnsInLeaf, OFFSET_COMPARATOR);
- int columnOrdinal = 0;
- for (int i = 0; i < numberOfColumns; i++) {
- int columnIndex = getColumnIndexFromPair(offsetColumnIndexPairs[i]);
- int offset = getOffsetFromPair(offsetColumnIndexPairs[i]);
- int nextOffset = getOffsetFromPair(offsetColumnIndexPairs[i + 1]);
+ int columnOrdinal = 0;
+ for (int i = 0; i < numberOfPresentColumnsInLeaf; i++) {
+ if (offsetColumnIndexPairs[i] == 0) {
+ //Any requested column's offset can't be zero
+ //In case a column is not being present in the accessed pageZero segments, it will be defaulted to 0
+ continue;
+ }
+ int columnIndex = getColumnIndexFromPair(offsetColumnIndexPairs[i]);
+ int offset = getOffsetFromPair(offsetColumnIndexPairs[i]);
+ int nextOffset = getOffsetFromPair(offsetColumnIndexPairs[i + 1]);
- // Compute the column's length in bytes (set 0 for PKs)
- int length = columnIndex < numberOfPrimaryKeys ? 0 : nextOffset - offset;
- lengths[columnIndex] = length;
+ // Compute the column's length in bytes (set 0 for PKs)
+ int length = columnIndex < numberOfPrimaryKeys ? 0 : nextOffset - offset;
+ // In case of sparse columns, few columnIndexes can be greater than the total sparse column count.
+ ensureCapacity(columnIndex);
+ lengths[columnIndex] = length;
- // Get start page ID (given the computed length above)
- int startPageId = getColumnStartPageIndex(columnIndex);
- // Get the number of pages (given the computed length above)
- int numberOfPages = getColumnNumberOfPages(columnIndex);
+ // Get start page ID (given the computed length above)
+ int startPageId = getColumnStartPageIndex(columnIndex);
+ // Get the number of pages (given the computed length above)
+ int numberOfPages = getColumnNumberOfPages(columnIndex);
- if (columnIndex >= numberOfPrimaryKeys && requestedColumns.get(columnIndex)) {
- // Set column index
- columnsOrder[columnOrdinal++] = columnIndex;
- // Compute cloud-only and evictable pages
- setCloudOnlyAndEvictablePages(columnIndex, cloudOnlyColumns, evictableColumns, startPageId,
- numberOfPages);
- // A requested column. Keep its pages as requested
- continue;
+ if (columnIndex >= numberOfPrimaryKeys && requestedColumns.get(columnIndex)) {
+ // Set column index
+ columnsOrder[columnOrdinal++] = columnIndex;
+ // Compute cloud-only and evictable pages
+ setCloudOnlyAndEvictablePages(columnIndex, cloudOnlyColumns, evictableColumns, startPageId,
+ numberOfPages);
+ // A requested column. Keep its pages as requested
+ continue;
+ }
+
+ // Mark the page as non-evictable
+ for (int j = startPageId; j < startPageId + numberOfPages; j++) {
+ nonEvictablePages.set(j);
+ }
}
- // Mark the page as non-evictable
- for (int j = startPageId; j < startPageId + numberOfPages; j++) {
- nonEvictablePages.set(j);
- }
+ // Bound the nonRequestedPages to the number of pages in the mega leaf node
+ nonEvictablePages.set(leafFrame.getMegaLeafNodeNumberOfPages());
+ // to indicate the end
+ columnsOrder[columnOrdinal] = -1;
+ } finally {
+ //Unpin the not required segment pages
+ leafFrame.unPinNotRequiredPageZeroSegments();
}
-
- // Bound the nonRequestedPages to the number of pages in the mega leaf node
- nonEvictablePages.set(leafFrame.getMegaLeafNodeNumberOfPages());
- // to indicate the end
- columnsOrder[columnOrdinal] = -1;
}
/**
@@ -166,7 +178,7 @@
* @param columnIndex column index
* @return pageID
*/
- public int getColumnStartPageIndex(int columnIndex) {
+ public int getColumnStartPageIndex(int columnIndex) throws HyracksDataException {
int pageSize = leafFrame.getBuffer().capacity();
return getColumnPageIndex(leafFrame.getColumnOffset(columnIndex), pageSize);
}
@@ -177,7 +189,7 @@
* @param columnIndex column index
* @return number of pages
*/
- public int getColumnNumberOfPages(int columnIndex) {
+ public int getColumnNumberOfPages(int columnIndex) throws HyracksDataException {
int pageSize = leafFrame.getBuffer().capacity();
int offset = getColumnStartOffset(leafFrame.getColumnOffset(columnIndex), pageSize);
int firstBufferLength = pageSize - offset;
@@ -278,6 +290,12 @@
}
}
+ private void ensureCapacity(int columnIndex) {
+ if (columnIndex >= lengths.length) {
+ lengths = IntArrays.grow(lengths, columnIndex + 1);
+ }
+ }
+
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
@@ -292,8 +310,18 @@
for (int i = 0; i < leafFrame.getNumberOfColumns(); i++) {
builder.append(String.format("%03d", i));
builder.append(":");
- int startPageId = getColumnStartPageIndex(i);
- int columnPagesCount = getColumnNumberOfPages(i);
+ int startPageId = 0;
+ try {
+ startPageId = getColumnStartPageIndex(i);
+ } catch (HyracksDataException e) {
+ throw new RuntimeException(e);
+ }
+ int columnPagesCount = 0;
+ try {
+ columnPagesCount = getColumnNumberOfPages(i);
+ } catch (HyracksDataException e) {
+ throw new RuntimeException(e);
+ }
printColumnPages(builder, numberOfPages, startPageId, columnPagesCount);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/IColumnReadContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/IColumnReadContext.java
index 86a650b..e37b9b1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/IColumnReadContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/IColumnReadContext.java
@@ -43,6 +43,9 @@
ICachedPage pinNext(ColumnBTreeReadLeafFrame leafFrame, IBufferCache bufferCache, int fileId)
throws HyracksDataException;
+ void preparePageZeroSegments(ColumnBTreeReadLeafFrame leafFrame, IBufferCache bufferCache, int fileId)
+ throws HyracksDataException;
+
/**
* Prepare the columns' pages
* Notes:
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
index 8d64eec..181aa06 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
@@ -131,6 +131,27 @@
}
@Override
+ public void preparePageZeroSegments(ColumnBTreeReadLeafFrame leafFrame, IBufferCache bufferCache, int fileId)
+ throws HyracksDataException {
+ if (leafFrame.getNumberOfPageZeroSegments() <= 1) { // don't need to include the zeroth segment
+ return;
+ }
+
+ // pin the required page segments
+ // mergedPageRanges.clear();
+ int pageZeroId = leafFrame.getPageId();
+ // Pinning all the segments of the page zero
+ // as the column eviction logic is based on the length of the columns which
+ // gets evaluated from the page zero segments.
+ BitSet pageZeroSegmentRanges =
+ leafFrame.markRequiredPageZeroSegments(projectedColumns, pageZeroId, operation == MERGE);
+ // will unpin the non-required segments after columnRanges.reset()
+ // can we do lazily?
+ int numberOfPageZeroSegments = leafFrame.getNumberOfPageZeroSegments();
+ pinAll(fileId, pageZeroId, numberOfPageZeroSegments - 1, bufferCache);
+ }
+
+ @Override
public void prepareColumns(ColumnBTreeReadLeafFrame leafFrame, IBufferCache bufferCache, int fileId)
throws HyracksDataException {
if (leafFrame.getTupleCount() == 0) {
@@ -139,9 +160,12 @@
columnRanges.reset(leafFrame, projectedColumns, plan, cloudOnlyColumns);
int pageZeroId = leafFrame.getPageId();
+ int numberOfPageZeroSegments = leafFrame.getNumberOfPageZeroSegments();
if (operation == MERGE) {
- pinAll(fileId, pageZeroId, leafFrame.getMegaLeafNodeNumberOfPages() - 1, bufferCache);
+ // will contain column pages along with page zero segments
+ pinAll(fileId, pageZeroId + numberOfPageZeroSegments - 1,
+ leafFrame.getMegaLeafNodeNumberOfPages() - numberOfPageZeroSegments, bufferCache);
} else {
pinProjected(fileId, pageZeroId, bufferCache);
}
@@ -198,6 +222,36 @@
mergedPageRanges.pin(columnCtx, bufferCache, fileId, pageZeroId);
}
+ private void mergePageZeroSegmentRanges(BitSet pageZeroSegmentRanges) {
+ // Since the 0th segment is already pinned, we can skip it
+ pageZeroSegmentRanges.clear(0);
+ if (pageZeroSegmentRanges.cardinality() == 0) {
+ // No page zero segments, nothing to merge
+ return;
+ }
+
+ int start = -1;
+ int prev = -1;
+
+ int current = pageZeroSegmentRanges.nextSetBit(0);
+ while (current >= 0) {
+ if (start == -1) {
+ // Start of a new range
+ start = current;
+ } else if (current != prev + 1) {
+ // Discontinuous: close the current range
+ mergedPageRanges.addRange(start, prev);
+ start = current;
+ }
+
+ prev = current;
+ current = pageZeroSegmentRanges.nextSetBit(current + 1);
+ }
+
+ // Close the final range
+ mergedPageRanges.addRange(start, prev);
+ }
+
@Override
public void release(IBufferCache bufferCache) throws HyracksDataException {
// Release might differ in the future if prefetching is supported
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
index 11275a4..5917f65 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
@@ -76,7 +76,14 @@
}
@Override
+ public void preparePageZeroSegments(ColumnBTreeReadLeafFrame leafFrame, IBufferCache bufferCache, int fileId)
+ throws HyracksDataException {
+ // NoOp
+ }
+
+ @Override
public void prepareColumns(ColumnBTreeReadLeafFrame leafFrame, IBufferCache bufferCache, int fileId) {
+ // If there are page segments, they are expected to be present in flash cache.
// NoOp
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
index d12f649..5b837a1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
@@ -20,6 +20,7 @@
import static org.apache.hyracks.storage.am.lsm.common.util.LSMComponentIdUtils.isMergedComponent;
+import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -58,6 +59,7 @@
private final IntSet indexedColumns;
private final ISweepClock clock;
private final int evictionPlanReevaluationThreshold;
+ private final List<ICachedPage> pageZeroSegmentsTempHolder;
private int numberOfColumns;
private long lastAccess;
private int maxSize;
@@ -78,6 +80,7 @@
plan = new BitSet();
reevaluatedPlan = new BitSet();
punchableThreshold = INITIAL_PUNCHABLE_THRESHOLD;
+ pageZeroSegmentsTempHolder = new ArrayList<>();
this.evictionPlanReevaluationThreshold = evictionPlanReevaluationThreshold;
}
@@ -215,12 +218,19 @@
ICachedPage page = bufferCache.pin(dpid);
try {
leafFrame.setPage(page);
+ pageZeroSegmentsTempHolder.clear();
+ leafFrame.pinPageZeroSegments(columnBTree.getFileId(), columnBTree.getBulkloadLeafStart(),
+ pageZeroSegmentsTempHolder, bufferCache, null);
ranges.reset(leafFrame);
for (int i = 0; i < leafFrame.getNumberOfColumns(); i++) {
sizes[i] = Math.max(sizes[i], ranges.getColumnLength(i));
maxSize = Math.max(maxSize, sizes[i]);
}
} finally {
+ // unpin the segment pages
+ for (ICachedPage segmentPage : pageZeroSegmentsTempHolder) {
+ bufferCache.unpin(segmentPage);
+ }
bufferCache.unpin(page);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
index 9fc3b8d..ba4376f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
@@ -38,6 +38,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.context.IBufferCacheReadContext;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.util.StorageUtil;
@@ -50,11 +51,13 @@
private final ColumnSweepLockInfo lockedColumns;
private final ColumnRanges ranges;
private final List<ILSMDiskComponent> sweepableComponents;
+ private final List<ICachedPage> segmentPagesTempHolder;
public ColumnSweeper(int numberOfPrimaryKeys) {
lockedColumns = new ColumnSweepLockInfo();
ranges = new ColumnRanges(numberOfPrimaryKeys);
sweepableComponents = new ArrayList<>();
+ segmentPagesTempHolder = new ArrayList<>();
}
public long sweep(BitSet plan, SweepContext context, IColumnTupleProjector sweepProjector)
@@ -162,6 +165,7 @@
int freedSpace = 0;
context.open(fileId);
try {
+ Throwable failure = null;
while (nextPageId >= 0) {
if (context.stopSweeping()) {
// Exit as the index is being dropped
@@ -175,16 +179,35 @@
columnsLocked = page0.trySweepLock(lockedColumns);
if (columnsLocked) {
leafFrame.setPage(page0);
+ leafFrame.pinPageZeroSegments(fileId, leafFrame.getPageId(), segmentPagesTempHolder,
+ context.getBufferCache(), SweepBufferCacheReadContext.INSTANCE);
ranges.reset(leafFrame, plan);
freedSpace += punchHoles(context, leafFrame);
}
+ } catch (Throwable th) {
+ failure = th;
} finally {
if (columnsLocked) {
page0.sweepUnlock();
}
+ // unpin segment pages
+ for (ICachedPage cachedPage : segmentPagesTempHolder) {
+ try {
+ context.unpin(cachedPage, bcOpCtx);
+ } catch (Throwable e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
+ }
context.unpin(page0, bcOpCtx);
}
}
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
} finally {
context.close();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResource.java
index 7ea4e35..65e19bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/dataflow/LSMColumnBTreeLocalResource.java
@@ -30,6 +30,8 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.INullIntrospector;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManagerFactory;
@@ -83,15 +85,16 @@
@Override
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
+ NCConfig config = ((NodeControllerService) serviceCtx.getControllerService()).getConfiguration();
IIOManager ioManager = storageManager.getIoManager(serviceCtx);
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
ioOpCallbackFactory.initialize(serviceCtx, this);
pageWriteCallbackFactory.initialize(serviceCtx, this);
IDiskCacheMonitoringService diskCacheService = storageManager.getDiskCacheMonitoringService(serviceCtx);
- return LSMColumnBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx),
- typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
- mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+ return LSMColumnBTreeUtil.createLSMTree(config, ioManager, vbcs, file,
+ storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields,
+ bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory, pageWriteCallbackFactory, btreeFields, metadataPageManagerFactory, false,
serviceCtx.getTracer(), compressorDecompressorFactory, nullTypeTraits, nullIntrospector,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
index a069a1c..f03195b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
@@ -67,13 +67,16 @@
public static final int HEADER_SIZE = NEXT_LEAF_OFFSET + 4;
protected final ITreeIndexTupleWriter rowTupleWriter;
+ protected final IColumnPageZeroWriterFlavorSelector pageZeroWriterFlavorSelector;
protected MultiComparator cmp;
protected ICachedPage page;
protected ByteBuffer buf;
- AbstractColumnBTreeLeafFrame(ITreeIndexTupleWriter rowTupleWriter) {
+ AbstractColumnBTreeLeafFrame(ITreeIndexTupleWriter rowTupleWriter,
+ IColumnPageZeroWriterFlavorSelector columnPageZeroWriterFlavorSelector) {
this.rowTupleWriter = rowTupleWriter;
+ this.pageZeroWriterFlavorSelector = columnPageZeroWriterFlavorSelector;
}
/* ****************************************************************************
@@ -97,9 +100,13 @@
// Duplicate to avoid interference when scanning the dataset twice
this.buf = page.getBuffer().duplicate();
buf.clear();
- buf.position(HEADER_SIZE);
+ resetPageZeroReader();
}
+ protected void resetPageZeroReader() {
+ buf.position(HEADER_SIZE);
+ };
+
@Override
public final ICachedPage getPage() {
return page;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
index a7a07be..a863d13 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.btree.impls.DiskBTree;
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
@@ -51,12 +52,14 @@
throw new IllegalAccessError("Missing write column metadata");
}
- public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, IPageWriteCallback callback,
- IColumnMetadata columnMetadata, IColumnWriteContext writeContext) throws HyracksDataException {
+ public IIndexBulkLoader createBulkLoader(NCConfig storageConfig, float fillFactor, boolean verifyInput,
+ IPageWriteCallback callback, IColumnMetadata columnMetadata, IColumnWriteContext writeContext)
+ throws HyracksDataException {
ColumnBTreeLeafFrameFactory columnLeafFrameFactory = (ColumnBTreeLeafFrameFactory) leafFrameFactory;
ColumnBTreeWriteLeafFrame writeLeafFrame =
columnLeafFrameFactory.createWriterFrame(columnMetadata, writeContext);
- return new ColumnBTreeBulkloader(fillFactor, verifyInput, callback, this, writeLeafFrame, writeContext);
+ return new ColumnBTreeBulkloader(storageConfig, fillFactor, verifyInput, callback, this, writeLeafFrame,
+ writeContext);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java
index 114ac05..9ca57dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java
@@ -23,6 +23,7 @@
import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTreeNSMBulkLoader;
import org.apache.hyracks.storage.am.btree.impls.BTreeSplitKey;
@@ -49,11 +50,14 @@
public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements IColumnWriteMultiPageOp {
private static final Logger LOGGER = LogManager.getLogger();
private final List<CachedPage> columnsPages;
+ private final List<ICachedPage> pageZeroSegments; // contains from 1st segment to the last segment of page0
private final List<CachedPage> tempConfiscatedPages;
private final ColumnBTreeWriteLeafFrame columnarFrame;
private final AbstractColumnTupleWriter columnWriter;
private final ISplitKey lowKey;
private final IColumnWriteContext columnWriteContext;
+ private final int maxColumnsInPageZerothSegment;
+ private final IColumnPageZeroWriter.ColumnPageZeroWriterType pageZeroWriterType;
private boolean setLowKey;
private int tupleCount;
@@ -61,14 +65,17 @@
private int numberOfLeafNodes;
private int numberOfPagesInCurrentLeafNode;
private int maxNumberOfPagesForAColumn;
+ private int maxNumberOfPageZeroSegments; // Exclude the zeroth segment
private int maxNumberOfPagesInALeafNode;
private int maxTupleCount;
private int lastRequiredFreeSpace;
- public ColumnBTreeBulkloader(float fillFactor, boolean verifyInput, IPageWriteCallback callback, ITreeIndex index,
- ITreeIndexFrame leafFrame, IBufferCacheWriteContext writeContext) throws HyracksDataException {
+ public ColumnBTreeBulkloader(NCConfig storageConfig, float fillFactor, boolean verifyInput,
+ IPageWriteCallback callback, ITreeIndex index, ITreeIndexFrame leafFrame,
+ IBufferCacheWriteContext writeContext) throws HyracksDataException {
super(fillFactor, verifyInput, callback, index, leafFrame, writeContext);
columnsPages = new ArrayList<>();
+ pageZeroSegments = new ArrayList<>();
tempConfiscatedPages = new ArrayList<>();
columnWriteContext = (IColumnWriteContext) writeContext;
columnarFrame = (ColumnBTreeWriteLeafFrame) leafFrame;
@@ -78,10 +85,16 @@
lowKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
setLowKey = true;
+ // Writer config
+ maxColumnsInPageZerothSegment = storageConfig.getStorageMaxColumnsInZerothSegment();
+ pageZeroWriterType = IColumnPageZeroWriter.ColumnPageZeroWriterType
+ .valueOf(storageConfig.getStoragePageZeroWriter().toUpperCase());
+
// For logging. Starts with 1 for page0
numberOfPagesInCurrentLeafNode = 1;
maxNumberOfPagesForAColumn = 0;
maxNumberOfPagesInALeafNode = 0;
+ maxNumberOfPageZeroSegments = 0;
numberOfLeafNodes = 1;
maxTupleCount = 0;
lastRequiredFreeSpace = 0;
@@ -93,6 +106,10 @@
writeFullLeafPage();
confiscateNewLeafPage();
}
+ if (tupleCount == 0) {
+ //Since we are writing the first tuple, we need to estimate the number of columns.
+ columnWriter.updateColumnMetadataForCurrentTuple(tuple);
+ }
//Save the key of the last inserted tuple
setMinMaxKeys(tuple);
columnWriter.writeTuple(tuple);
@@ -106,17 +123,20 @@
private boolean isFull(ITupleReference tuple) throws HyracksDataException {
if (tupleCount == 0) {
+ columnWriter.updateColumnMetadataForCurrentTuple(tuple);
+ // this is for non-adaptive case.
+ columnWriter.setWriterType(pageZeroWriterType);
return false;
} else if (tupleCount >= columnWriter.getMaxNumberOfTuples()) {
//We reached the maximum number of tuples
return true;
}
- int requiredFreeSpace = AbstractColumnBTreeLeafFrame.HEADER_SIZE;
//Columns' Offsets
columnWriter.updateColumnMetadataForCurrentTuple(tuple);
- requiredFreeSpace += columnWriter.getColumnOffsetsSize(true);
+ int requiredFreeSpace = columnWriter.getPageZeroWriterOccupiedSpace(maxColumnsInPageZerothSegment,
+ columnarFrame.getBuffer().capacity(), true, pageZeroWriterType);
//Occupied space from previous writes
- requiredFreeSpace += columnWriter.getOccupiedSpace();
+ requiredFreeSpace += columnWriter.getPrimaryKeysEstimatedSize();
//min and max tuples' sizes
requiredFreeSpace += lowKey.getTuple().getTupleSize() + getSplitKeySize(tuple);
//New tuple required space
@@ -139,7 +159,8 @@
if (tupleCount > 0) {
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0);
try {
- columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple());
+ columnarFrame.flush(columnWriter, tupleCount, maxColumnsInPageZerothSegment, lowKey.getTuple(),
+ splitKey.getTuple(), this);
} catch (Exception e) {
logState(e);
throw e;
@@ -168,7 +189,8 @@
if (tupleCount > 0) {
//We need to flush columns to confiscate all columns pages first before calling propagateBulk
try {
- columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple());
+ columnarFrame.flush(columnWriter, tupleCount, maxColumnsInPageZerothSegment, lowKey.getTuple(),
+ splitKey.getTuple(), this);
} catch (Exception e) {
logState(e);
throw e;
@@ -185,7 +207,7 @@
* Write columns' pages first to ensure they (columns' pages) are written before pageZero.
* It ensures pageZero does not land in between columns' pages if compression is enabled
*/
- writeColumnsPages();
+ writeColumnAndSegmentPages();
//Then write page0
write(leafFrontier.page);
@@ -214,10 +236,34 @@
* Write columns' pages first to ensure they (columns' pages) are written before pageZero.
* It ensures pageZero does not land in between columns' pages if compression is enabled
*/
- writeColumnsPages();
+ writeColumnAndSegmentPages();
super.writeLastLeaf(page);
}
+ private void writeColumnAndSegmentPages() throws HyracksDataException {
+ for (ICachedPage c : columnsPages) {
+ write(c);
+ }
+
+ // For logging
+ int numberOfPagesInPersistedColumn = columnsPages.size();
+ maxNumberOfPagesForAColumn = Math.max(maxNumberOfPagesForAColumn, numberOfPagesInPersistedColumn);
+ numberOfPagesInCurrentLeafNode += numberOfPagesInPersistedColumn;
+ columnsPages.clear();
+
+ // persist page zero segments from 1 to the last segment
+ for (ICachedPage page : pageZeroSegments) {
+ write(page);
+ }
+
+ int numberOfPageZeroSegments = pageZeroSegments.size();
+ maxNumberOfPageZeroSegments = Math.max(maxNumberOfPageZeroSegments, numberOfPageZeroSegments);
+ pageZeroSegments.clear();
+
+ // Indicate to the columnWriteContext that all columns were persisted
+ columnWriteContext.columnsPersisted();
+ }
+
private void writeColumnsPages() throws HyracksDataException {
for (ICachedPage c : columnsPages) {
write(c);
@@ -239,6 +285,10 @@
bufferCache.returnPage(page, false);
}
+ for (ICachedPage page : pageZeroSegments) {
+ bufferCache.returnPage(page, false);
+ }
+
for (ICachedPage page : tempConfiscatedPages) {
bufferCache.returnPage(page, false);
}
@@ -286,6 +336,15 @@
}
@Override
+ public ByteBuffer confiscatePageZeroPersistent() throws HyracksDataException {
+ int pageId = freePageManager.takePage(metaFrame);
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, pageId);
+ CachedPage page = (CachedPage) bufferCache.confiscatePage(dpid);
+ pageZeroSegments.add(page);
+ return page.getBuffer();
+ }
+
+ @Override
public void persist() throws HyracksDataException {
writeColumnsPages();
}
@@ -308,9 +367,9 @@
// number of tuples processed for the current leaf
state.put("currentLeafTupleCount", tupleCount);
// number of columns
- state.put("currentLeafColumnCount", columnWriter.getNumberOfColumns(false));
+ state.put("currentLeafColumnCount", columnWriter.getAbsoluteNumberOfColumns(false));
// number of columns including current tuple
- state.put("currentColumnCount", columnWriter.getNumberOfColumns(true));
+ state.put("currentColumnCount", columnWriter.getAbsoluteNumberOfColumns(true));
state.put("lastRequiredFreeSpace", lastRequiredFreeSpace);
state.put("splitKeyTupleSize", splitKey.getTuple().getTupleSize());
state.put("splitKeyTupleSizeByTupleWriter", tupleWriter.bytesRequired(splitKey.getTuple()));
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeRangeSearchCursor.java
index 2669d4b..6bcbd0f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeRangeSearchCursor.java
@@ -92,8 +92,9 @@
do {
page0 = context.pinNext(frame, bufferCache, fileId);
stats.getPageCounter().update(1);
- context.prepareColumns(frame, bufferCache, fileId);
+ context.preparePageZeroSegments(frame, bufferCache, fileId);
frameTuple.newPage();
+ context.prepareColumns(frame, bufferCache, fileId);
setCursorPosition();
nextLeafPage = frame.getNextLeaf();
} while (frame.getTupleCount() == 0 && nextLeafPage > 0);
@@ -131,8 +132,9 @@
frame.setPage(page0);
frame.setMultiComparator(originalKeyCmp);
if (frame.getTupleCount() > 0) {
- context.prepareColumns(frame, bufferCache, fileId);
+ context.preparePageZeroSegments(frame, bufferCache, fileId);
frameTuple.newPage();
+ context.prepareColumns(frame, bufferCache, fileId);
initCursorPosition(searchPred);
} else {
yieldFirstCall = false;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
index 1b4d09d..3307423 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
@@ -18,6 +18,10 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
@@ -25,29 +29,54 @@
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
import org.apache.hyracks.storage.common.buffercache.CachedPage;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.context.IBufferCacheReadContext;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
public final class ColumnBTreeReadLeafFrame extends AbstractColumnBTreeLeafFrame {
private final AbstractColumnTupleReader columnarTupleReader;
private final ITreeIndexTupleReference leftMostTuple;
private final ITreeIndexTupleReference rightMostTuple;
+ private IColumnPageZeroReader columnPageZeroReader;
public ColumnBTreeReadLeafFrame(ITreeIndexTupleWriter rowTupleWriter,
AbstractColumnTupleReader columnarTupleReader) {
- super(rowTupleWriter);
+ super(rowTupleWriter, columnarTupleReader.getPageZeroWriterFlavorSelector());
this.columnarTupleReader = columnarTupleReader;
leftMostTuple = rowTupleWriter.createTupleReference();
rightMostTuple = rowTupleWriter.createTupleReference();
}
@Override
+ protected void resetPageZeroReader() {
+ columnPageZeroReader = pageZeroWriterFlavorSelector.createPageZeroReader(getFlagByte(), buf.capacity());
+ columnPageZeroReader.reset(buf);
+ buf.position(columnPageZeroReader.getHeaderSize());
+ }
+
+ public void pinPageZeroSegments(int fileId, int pageZeroId, List<ICachedPage> segmentPages,
+ IBufferCache bufferCache, IBufferCacheReadContext bcOpCtx) throws HyracksDataException {
+ // pins all the segments, used by the column planner and sweeper
+ int numberOfPageSegments = getNumberOfPageZeroSegments();
+ for (int i = 1; i < numberOfPageSegments; i++) {
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, pageZeroId + i);
+ if (bcOpCtx != null) {
+ segmentPages.add(bufferCache.pin(dpid, bcOpCtx));
+ } else {
+ segmentPages.add(bufferCache.pin(dpid));
+ }
+ }
+ }
+
+ @Override
public ITupleReference getLeftmostTuple() {
if (getTupleCount() == 0) {
return null;
}
leftMostTuple.setFieldCount(cmp.getKeyFieldCount());
- leftMostTuple.resetByTupleOffset(buf.array(), buf.getInt(LEFT_MOST_KEY_OFFSET));
+ leftMostTuple.resetByTupleOffset(buf.array(), columnPageZeroReader.getLeftMostKeyOffset());
return leftMostTuple;
}
@@ -58,44 +87,86 @@
}
rightMostTuple.setFieldCount(cmp.getKeyFieldCount());
- rightMostTuple.resetByTupleOffset(buf.array(), buf.getInt(RIGHT_MOST_KEY_OFFSET));
+ rightMostTuple.resetByTupleOffset(buf.array(), columnPageZeroReader.getRightMostKeyOffset());
return rightMostTuple;
}
+ public void getAllColumns(BitSet presentColumns) {
+ columnPageZeroReader.getAllColumns(presentColumns);
+ }
+
public IColumnTupleIterator createTupleReference(int index, IColumnReadMultiPageOp multiPageOp) {
return columnarTupleReader.createTupleIterator(this, index, multiPageOp);
}
@Override
public int getTupleCount() {
- return buf.getInt(Constants.TUPLE_COUNT_OFFSET);
+ return columnPageZeroReader.getTupleCount();
}
public int getPageId() {
return BufferedFileHandle.getPageId(((CachedPage) page).getDiskPageId());
}
- public int getNumberOfColumns() {
- return buf.getInt(NUMBER_OF_COLUMNS_OFFSET);
+ public int getNumberOfPageZeroSegments() {
+ return columnPageZeroReader.getNumberOfPageZeroSegments();
}
- public int getColumnOffset(int columnIndex) {
- if (columnIndex >= getNumberOfColumns()) {
+ public int getNumberOfColumns() {
+ return columnPageZeroReader.getNumberOfPresentColumns();
+ }
+
+ public int getColumnOffset(int columnIndex) throws HyracksDataException {
+ // update the exception message.
+ if (!columnPageZeroReader.isValidColumn(columnIndex)) {
+ printPageZeroReaderInfo();
throw new IndexOutOfBoundsException(columnIndex + " >= " + getNumberOfColumns());
}
- return columnarTupleReader.getColumnOffset(buf, columnIndex);
+ return columnPageZeroReader.getColumnOffset(columnIndex);
+ }
+
+ public boolean isValidColumn(int columnIndex) throws HyracksDataException {
+ return columnPageZeroReader.isValidColumn(columnIndex);
}
public int getNextLeaf() {
- return buf.getInt(NEXT_LEAF_OFFSET);
+ return columnPageZeroReader.getNextLeaf();
}
public int getMegaLeafNodeLengthInBytes() {
- return buf.getInt(MEGA_LEAF_NODE_LENGTH);
+ return columnPageZeroReader.getMegaLeafNodeLengthInBytes();
+ }
+
+ public int getHeaderSize() {
+ return columnPageZeroReader.getHeaderSize();
+ }
+
+ public int getPageZeroSegmentCount() {
+ return columnPageZeroReader.getNumberOfPageZeroSegments();
+ }
+
+ // flag needs to be directly accessed from the buffer, as this will be used to choose the pageReader
+ public byte getFlagByte() {
+ return buf.get(FLAG_OFFSET);
+ }
+
+ public void skipFilters() {
+ columnPageZeroReader.skipFilters();
+ }
+
+ public void skipColumnOffsets() {
+ columnPageZeroReader.skipColumnOffsets();
+ }
+
+ public IColumnPageZeroReader getColumnPageZeroReader() {
+ return columnPageZeroReader;
}
public int getMegaLeafNodeNumberOfPages() {
- return (int) Math.ceil((double) getMegaLeafNodeLengthInBytes() / buf.capacity());
+ // the denominator should ideally be the bufferCache pageSize, but
+ // in the current way, the pageZeroCapacity = bufferCache's pageSize.
+ // May be, needs to be changed in the future, to point to the bufferCache's pageSize.
+ return (int) Math.ceil((double) getMegaLeafNodeLengthInBytes() / columnPageZeroReader.getPageZeroCapacity());
}
public ColumnBTreeReadLeafFrame createCopy() {
@@ -106,4 +177,24 @@
public ITreeIndexTupleReference createTupleReference() {
throw new IllegalArgumentException("Use createTupleReference(int)");
}
+
+ public int populateOffsetColumnIndexPairs(long[] offsetColumnIndexPairs) {
+ return columnPageZeroReader.populateOffsetColumnIndexPairs(offsetColumnIndexPairs);
+ }
+
+ public BitSet getPageZeroSegmentsPages() {
+ return columnPageZeroReader.getPageZeroSegmentsPages();
+ }
+
+ public BitSet markRequiredPageZeroSegments(BitSet projectedColumns, int pageZeroId, boolean markAll) {
+ return columnPageZeroReader.markRequiredPageSegments(projectedColumns, pageZeroId, markAll);
+ }
+
+ public void unPinNotRequiredPageZeroSegments() throws HyracksDataException {
+ columnPageZeroReader.unPinNotRequiredPageZeroSegments();
+ }
+
+ public void printPageZeroReaderInfo() {
+ columnPageZeroReader.printPageZeroReaderInfo();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java
index 1fc5712..acee2d5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -30,7 +31,7 @@
public ColumnBTreeWriteLeafFrame(ITreeIndexTupleWriter rowTupleWriter,
AbstractColumnTupleWriter columnTupleWriter) {
- super(rowTupleWriter);
+ super(rowTupleWriter, columnTupleWriter.getColumnPageZeroWriterFlavorSelector());
this.columnTupleWriter = columnTupleWriter;
}
@@ -47,25 +48,12 @@
buf.putInt(NEXT_LEAF_OFFSET, -1);
}
- void flush(AbstractColumnTupleWriter columnWriter, int numberOfTuples, ITupleReference minKey,
- ITupleReference maxKey) throws HyracksDataException {
- // Prepare the space for writing the columns' information such as the primary keys
- buf.position(HEADER_SIZE);
- // Flush the columns to persistence pages and write the length of the mega leaf node in pageZero
- buf.putInt(MEGA_LEAF_NODE_LENGTH, columnWriter.flush(buf));
-
- // Write min and max keys
- int offset = buf.position();
- buf.putInt(LEFT_MOST_KEY_OFFSET, offset);
- offset += rowTupleWriter.writeTuple(minKey, buf.array(), offset);
- buf.putInt(RIGHT_MOST_KEY_OFFSET, offset);
- rowTupleWriter.writeTuple(maxKey, buf.array(), offset);
-
- // Write page information
- int numberOfColumns = columnWriter.getNumberOfColumns(false);
- buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples);
- buf.putInt(NUMBER_OF_COLUMNS_OFFSET, numberOfColumns);
- buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, columnWriter.getColumnOffsetsSize(false));
+ void flush(AbstractColumnTupleWriter columnWriter, int numberOfTuples, int zerothSegmentMaxColumns,
+ ITupleReference minKey, ITupleReference maxKey, IColumnWriteMultiPageOp multiPageOpRef)
+ throws HyracksDataException {
+ IColumnPageZeroWriter pageZeroWriter = pageZeroWriterFlavorSelector.getPageZeroWriter(multiPageOpRef,
+ zerothSegmentMaxColumns, getBuffer().capacity());
+ pageZeroWriter.flush(buf, numberOfTuples, minKey, maxKey, columnWriter, rowTupleWriter);
}
public AbstractColumnTupleWriter getColumnTupleWriter() {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
new file mode 100644
index 0000000..f57eba3
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+
+public interface IColumnPageZeroReader {
+
+ default void reset(ByteBuffer pageZeroBuf) {
+ reset(pageZeroBuf, AbstractColumnBTreeLeafFrame.HEADER_SIZE);
+ }
+
+ void reset(ByteBuffer pageZeroBuf, int headerSize);
+
+ int getColumnOffset(int columnIndex) throws HyracksDataException;
+
+ long getColumnFilterMin(int columnIndex) throws HyracksDataException;
+
+ long getColumnFilterMax(int columnIndex) throws HyracksDataException;
+
+ void skipFilters();
+
+ void skipColumnOffsets();
+
+ int getTupleCount();
+
+ int getLeftMostKeyOffset();
+
+ int getRightMostKeyOffset();
+
+ int getNumberOfPresentColumns();
+
+ int getRelativeColumnIndex(int columnIndex) throws HyracksDataException;
+
+ int getNextLeaf();
+
+ int getMegaLeafNodeLengthInBytes();
+
+ int getPageZeroCapacity();
+
+ boolean isValidColumn(int columnIndex) throws HyracksDataException;
+
+ void getAllColumns(BitSet presentColumns);
+
+ ByteBuffer getPageZeroBuf();
+
+ int populateOffsetColumnIndexPairs(long[] offsetColumnIndexPairs);
+
+ int getNumberOfPageZeroSegments();
+
+ BitSet getPageZeroSegmentsPages();
+
+ int getHeaderSize();
+
+ void resetStream(IColumnBufferProvider pageZeroSegmentBufferProvider) throws HyracksDataException;
+
+ BitSet markRequiredPageSegments(BitSet projectedColumns, int pageZeroId, boolean markAll);
+
+ void printPageZeroReaderInfo();
+
+ void unPinNotRequiredPageZeroSegments() throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroWriter.java
new file mode 100644
index 0000000..77fcee0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroWriter.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter;
+
+/**
+ * Interface for writing column metadata to page zero of a column page.
+ * This abstraction supports different page zero layouts including:
+ * - Default layout: stores all columns with fixed offsets
+ * - Sparse layout: stores only present columns with variable offsets
+ *
+ * The writer handles column offsets, filters (min/max values), and primary key data.
+ */
+public interface IColumnPageZeroWriter {
+
+ /** Flag code for default page zero writer */
+ byte DEFAULT_WRITER_FLAG = 0;
+
+ /** Flag code for sparse page zero writer */
+ byte SPARSE_WRITER_FLAG = 1;
+
+ byte MULTI_PAGE_DEFAULT_WRITER_FLAG = 2;
+
+ byte MULTI_PAGE_SPARSE_WRITER_FLAG = 3;
+
+ int MIN_COLUMN_SPACE = 4 + 16; // offset + filter size
+
+ enum ColumnPageZeroWriterType {
+ DEFAULT((byte) 2), // multi-page default page zero
+ SPARSE((byte) 3), // multi-page sparse page zero
+ ADAPTIVE((byte) -1); // adaptive writer that switches between default and sparse based on space efficiency
+
+ private final byte writerFlag;
+
+ ColumnPageZeroWriterType(byte writerFlag) {
+ this.writerFlag = writerFlag;
+ }
+
+ public byte getWriterFlag() {
+ return writerFlag;
+ }
+ }
+
+ /**
+ * Initializes the writer with page zero buffer and column information.
+ *
+ * @param presentColumns Array of column indexes that are present in this page
+ * @param numberOfColumns Total number of columns in the schema (may be larger than presentColumns)
+ */
+ void resetBasedOnColumns(int[] presentColumns, int numberOfColumns, int headerSize) throws HyracksDataException;
+
+ default void resetBasedOnColumns(int[] presentColumns, int numberOfColumns) throws HyracksDataException {
+ resetBasedOnColumns(presentColumns, numberOfColumns, AbstractColumnBTreeLeafFrame.HEADER_SIZE);
+ }
+
+ /**
+ * Returns the flag code that identifies this writer type.
+ * Used for selecting the appropriate reader during deserialization.
+ *
+ * @return flag code (DEFAULT_WRITER_FLAG for default, SPARSE_WRITER_FLAG for sparse)
+ */
+ byte flagCode();
+
+ /**
+ * Allocates space in page zero for column metadata.
+ * This includes space for column offsets, filters, and primary keys.
+ */
+ void allocateColumns();
+
+ /**
+ * Records the offset of a column's data within the page.
+ *
+ * @param absoluteColumnIndex The absolute column index in the schema
+ * @param relativeColumnIndex The relative column index within this page (for sparse layouts)
+ * @param offset The byte offset where the column's data begins
+ */
+ void putColumnOffset(int absoluteColumnIndex, int relativeColumnIndex, int offset) throws HyracksDataException;
+
+ /**
+ * Stores filter information (min/max values) for a column.
+ * This enables efficient filtering during query execution.
+ *
+ * @param relativeColumnIndex The relative column index within this page
+ * @param normalizedMinValue The normalized minimum value in the column
+ * @param normalizedMaxValue The normalized maximum value in the column
+ */
+ void putColumnFilter(int relativeColumnIndex, long normalizedMinValue, long normalizedMaxValue)
+ throws HyracksDataException;
+
+ /**
+ * Writes primary key column data to page zero.
+ * Primary keys are stored directly in page zero for fast access.
+ *
+ * @param primaryKeyWriters Array of writers containing primary key data
+ * @throws HyracksDataException If an error occurs during writing
+ */
+ void writePrimaryKeyColumns(IValuesWriter[] primaryKeyWriters) throws HyracksDataException;
+
+ /**
+ * Returns the number of columns handled by this writer.
+ * For default writers, this is the total schema size.
+ * For sparse writers, this is the number of present columns.
+ *
+ * @return number of columns
+ */
+ int getNumberOfColumns();
+
+ /**
+ * Determines whether a column should be included in ordered processing.
+ * This is particularly important for sparse columns where not all columns may be present.
+ *
+ * @param presentColumns Set of columns present in this page
+ * @param columnIndex The column index to check
+ * @param includeChildrenColumns Whether to include child columns for complex types
+ * @return true if the column should be included
+ */
+ boolean includeOrderedColumn(BitSet presentColumns, int columnIndex, boolean includeChildrenColumns);
+
+ /**
+ * Returns the page zero buffer being written to.
+ *
+ * @return the page zero buffer
+ */
+ int getPageZeroBufferCapacity();
+
+ /**
+ * Maps an absolute column index to a relative index within this page.
+ * For default layouts, this is typically an identity mapping.
+ * For sparse layouts, this maps to the position within the present columns array.
+ *
+ * @param columnIndex The absolute column index in the schema
+ * @return the relative column index within this page
+ */
+ int getRelativeColumnIndex(int columnIndex);
+
+ /**
+ * Returns the total size in bytes used for storing column offsets.
+ *
+ * @return size in bytes of column offset storage
+ */
+ int getColumnOffsetsSize();
+
+ void setPageZero(ByteBuffer pageZero);
+
+ void flush(ByteBuffer buf, int numberOfTuples, ITupleReference minKey, ITupleReference maxKey,
+ AbstractColumnTupleWriter columnWriter, ITreeIndexTupleWriter rowTupleWriter) throws HyracksDataException;
+
+ int getHeaderSize();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroWriterFlavorSelector.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroWriterFlavorSelector.java
new file mode 100644
index 0000000..0dc4d4d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroWriterFlavorSelector.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnWriteMultiPageOp;
+
+/**
+ * Strategy interface for selecting the optimal page zero writer implementation.
+ *
+ * This interface implements the Strategy pattern to choose between different
+ * page zero layouts based on space efficiency. The selector dynamically
+ * switches between default and sparse writers to minimize storage overhead.
+ *
+ * The selection is made by comparing the space requirements of both approaches
+ * and choosing the more efficient one for each specific data pattern.
+ */
+public interface IColumnPageZeroWriterFlavorSelector {
+
+ /**
+ * Evaluates and switches the page zero writer based on space efficiency.
+ * <p>
+ * This method compares the space requirements of both writer implementations
+ * and selects the one that uses less space. The decision is made dynamically
+ * for each batch of data to optimize storage utilization.
+ *
+ * @param spaceOccupiedByDefaultWriter Space in bytes required by the default writer
+ * @param spaceOccupiedBySparseWriter Space in bytes required by the sparse writer
+ */
+ void switchPageZeroWriterIfNeeded(int spaceOccupiedByDefaultWriter, int spaceOccupiedBySparseWriter);
+
+ byte getWriterFlag();
+
+ /**
+ * Creates the appropriate page zero reader for the given writer type.
+ * <p>
+ * This method is used during deserialization to create a reader that matches
+ * the writer type used during serialization. The flag identifies which
+ * layout was used.
+ *
+ * @param flag The flag code identifying the writer type (0=default, 1=sparse)
+ * @param capacity
+ * @return the appropriate reader instance
+ */
+ IColumnPageZeroReader createPageZeroReader(byte flag, int capacity);
+
+ void setPageZeroWriterFlag(byte writerFlag);
+
+ /**
+ * Returns the currently selected page zero writer instance.
+ *
+ * @return the writer instance selected by the most recent call to switchPageZeroWriterIfNeeded
+ */
+ IColumnPageZeroWriter getPageZeroWriter(IColumnWriteMultiPageOp multiPageOpRef, int zerothSegmentMaxColumns,
+ int maximumColumnPerPageSegment);
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IValuesWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IValuesWriter.java
new file mode 100644
index 0000000..191952c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IValuesWriter.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import java.io.OutputStream;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Base interface for writing column values to output streams.
+ *
+ * This interface provides a common abstraction for different types of column writers,
+ * enabling the page zero writer implementations to handle both primary key columns
+ * and regular column values uniformly. This abstraction is particularly important
+ * for the sparse column architecture where different writer types need to be
+ * handled consistently.
+ *
+ * Implementations include:
+ * - IColumnValuesWriter: For regular column data with additional metadata
+ * - Primary key writers: For key columns stored directly in page zero
+ */
+public interface IValuesWriter {
+ /**
+ * Flushes the column values to the specified output stream.
+ *
+ * This method writes the accumulated column data to the provided output stream.
+ *
+ * @param out The output stream to write the column data to
+ * @throws HyracksDataException If an error occurs during the write operation
+ */
+ void flush(OutputStream out) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
index d95678e..0ba4509 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.common.api.IExtendedModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManager;
@@ -64,7 +65,7 @@
*/
private IColumnMetadata columnMetadata;
- public LSMColumnBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public LSMColumnBTree(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache diskBufferCache,
ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
@@ -74,9 +75,9 @@
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
int[] btreeFields, ITracer tracer, IColumnManager columnManager, boolean atomic,
IColumnIndexDiskCacheManager diskCacheManager) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
- diskBufferCache, fileManager, componentFactory, bulkloadComponentFactory, null, null, null,
- bloomFilterFalsePositiveRate, fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler,
+ super(storageConfig, ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
+ deleteLeafFrameFactory, diskBufferCache, fileManager, componentFactory, bulkloadComponentFactory, null,
+ null, null, bloomFilterFalsePositiveRate, fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory, pageWriteCallbackFactory, true, true, btreeFields, null, true, false, tracer,
atomic);
this.columnManager = columnManager;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
index 6dde7e9..154aba9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
@@ -21,6 +21,7 @@
import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.btree.impls.BTree;
@@ -50,8 +51,8 @@
}
@Override
- public ChainedLSMDiskComponentBulkLoader createBulkLoader(ILSMIOOperation operation, float fillFactor,
- boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
+ public ChainedLSMDiskComponentBulkLoader createBulkLoader(NCConfig storageConfig, ILSMIOOperation operation,
+ float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
boolean cleanupEmptyComponent, IPageWriteCallback callback) throws HyracksDataException {
ChainedLSMDiskComponentBulkLoader chainedBulkLoader =
new ChainedLSMDiskComponentBulkLoader(operation, this, cleanupEmptyComponent);
@@ -60,7 +61,8 @@
chainedBulkLoader.addBulkLoader(createFilterBulkLoader());
}
//Add index bulkloader
- chainedBulkLoader.addBulkLoader(createColumnIndexBulkLoader(operation, fillFactor, verifyInput, callback));
+ chainedBulkLoader.addBulkLoader(
+ createColumnIndexBulkLoader(storageConfig, operation, fillFactor, verifyInput, callback));
if (numElementsHint > 0) {
chainedBulkLoader.addBulkLoader(createBloomFilterBulkLoader(numElementsHint, callback));
@@ -70,8 +72,8 @@
return chainedBulkLoader;
}
- private IChainedComponentBulkLoader createColumnIndexBulkLoader(ILSMIOOperation operation, float fillFactor,
- boolean verifyInput, IPageWriteCallback callback) throws HyracksDataException {
+ private IChainedComponentBulkLoader createColumnIndexBulkLoader(NCConfig storageConfig, ILSMIOOperation operation,
+ float fillFactor, boolean verifyInput, IPageWriteCallback callback) throws HyracksDataException {
LSMIOOperationType operationType = operation.getIOOperationType();
LSMColumnBTree lsmColumnBTree = (LSMColumnBTree) getLsmIndex();
ColumnBTree columnBTree = (ColumnBTree) getIndex();
@@ -91,8 +93,8 @@
int numberOfColumns = columnMetadata.getNumberOfColumns();
IColumnIndexDiskCacheManager diskCacheManager = lsmColumnBTree.getDiskCacheManager();
IColumnWriteContext writeContext = diskCacheManager.createWriteContext(numberOfColumns, operationType);
- IIndexBulkLoader bulkLoader =
- columnBTree.createBulkLoader(fillFactor, verifyInput, callback, columnMetadata, writeContext);
+ IIndexBulkLoader bulkLoader = columnBTree.createBulkLoader(storageConfig, fillFactor, verifyInput, callback,
+ columnMetadata, writeContext);
return new LSMColumnIndexBulkloader(bulkLoader, columnMetadata, getMetadata());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
index f079f51..de59836 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
@@ -18,8 +18,6 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples;
-import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.HEADER_SIZE;
-
import java.nio.ByteBuffer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -29,6 +27,7 @@
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -39,7 +38,8 @@
private static final Logger LOGGER = LogManager.getLogger();
private static final String UNSUPPORTED_OPERATION_MSG = "Operation is not supported for column tuples";
private final int componentIndex;
- private final ColumnBTreeReadLeafFrame frame;
+ protected final ColumnBTreeReadLeafFrame frame;
+ protected final IColumnBufferProvider pageZeroSegmentBufferProvider;
private final IColumnBufferProvider[] primaryKeyBufferProviders;
private final IColumnBufferProvider[] filterBufferProviders;
private final IColumnBufferProvider[] buffersProviders;
@@ -74,6 +74,7 @@
pinnedPages = new LongOpenHashSet();
int numberOfFilteredColumns = info.getNumberOfFilteredColumns();
filterBufferProviders = new IColumnBufferProvider[numberOfFilteredColumns];
+ pageZeroSegmentBufferProvider = new ColumnMultiPageZeroBufferProvider(multiPageOp, pinnedPages);
for (int i = 0; i < numberOfFilteredColumns; i++) {
int columnIndex = info.getFilteredColumnIndex(i);
if (columnIndex < 0) {
@@ -100,11 +101,12 @@
}
@Override
- public final void newPage() throws HyracksDataException {
+ public void newPage() throws HyracksDataException {
tupleIndex = 0;
ByteBuffer pageZero = frame.getBuffer();
+ // should not be needed, as it just been reset in step above
pageZero.clear();
- pageZero.position(HEADER_SIZE);
+ pageZero.position(frame.getHeaderSize());
int numberOfTuples = frame.getTupleCount();
@@ -114,16 +116,22 @@
provider.reset(frame);
startPrimaryKey(provider, i, numberOfTuples);
}
+
+ // if the pageZero segments > 1, reset the page zero segment buffer provider
+ if (frame.getPageZeroSegmentCount() > 1) {
+ IColumnPageZeroReader pageZeroReader = frame.getColumnPageZeroReader();
+ pageZeroSegmentBufferProvider.reset(frame);
+ pageZeroReader.resetStream(pageZeroSegmentBufferProvider);
+ }
}
@Override
public final void reset(int startIndex, int endIndex) throws HyracksDataException {
tupleIndex = startIndex;
this.endIndex = endIndex;
- ByteBuffer pageZero = frame.getBuffer();
int numberOfTuples = frame.getTupleCount();
//Start new page and check whether we should skip reading non-key columns or not
- boolean readColumnPages = startNewPage(pageZero, frame.getNumberOfColumns(), numberOfTuples);
+ boolean readColumnPages = startNewPage(numberOfTuples);
//Release previous pinned pages if any
unpinColumnsPages();
/*
@@ -196,8 +204,7 @@
protected abstract int setPrimaryKeysAt(int index, int skipCount) throws HyracksDataException;
- protected abstract boolean startNewPage(ByteBuffer pageZero, int numberOfColumns, int numberOfTuples)
- throws HyracksDataException;
+ protected abstract boolean startNewPage(int numberOfTuples) throws HyracksDataException;
protected abstract void startPrimaryKey(IColumnBufferProvider bufferProvider, int ordinal, int numberOfTuples)
throws HyracksDataException;
@@ -251,6 +258,9 @@
buffersProviders[i].releaseAll();
}
+ // release pageZero segment buffer provider
+ pageZeroSegmentBufferProvider.releaseAll();
+
maxNumberOfPinnedPages = Math.max(maxNumberOfPinnedPages, pinnedPages.size());
pinnedPages.clear();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiBufferProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiBufferProvider.java
index 22ef4f1..8737db3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiBufferProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiBufferProvider.java
@@ -35,13 +35,13 @@
import it.unimi.dsi.fastutil.longs.LongSet;
-public final class ColumnMultiBufferProvider implements IColumnBufferProvider {
+public class ColumnMultiBufferProvider implements IColumnBufferProvider {
private final int columnIndex;
private final IColumnReadMultiPageOp multiPageOp;
private final Queue<ICachedPage> pages;
private final LongSet pinnedPages;
- private int numberOfRemainingPages;
- private int startPage;
+ protected int numberOfRemainingPages;
+ protected int startPage;
private int startOffset;
private int length;
@@ -54,7 +54,7 @@
@Override
public void reset(ColumnBTreeReadLeafFrame frame) throws HyracksDataException {
- if (columnIndex >= frame.getNumberOfColumns()) {
+ if (!frame.isValidColumn(columnIndex)) {
numberOfRemainingPages = 0;
length = 0;
return;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiPageZeroBufferProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiPageZeroBufferProvider.java
new file mode 100644
index 0000000..c60ca79
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnMultiPageZeroBufferProvider.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.tuples;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Queue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.common.buffercache.CachedPage;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+import it.unimi.dsi.fastutil.ints.IntList;
+import it.unimi.dsi.fastutil.longs.LongSet;
+
+public class ColumnMultiPageZeroBufferProvider implements IColumnBufferProvider {
+ private static final BitSet EMPTY_SEGMENTS = new BitSet();
+ private final IColumnReadMultiPageOp multiPageOp;
+ private final LongSet pinnedPages;
+ private final List<ICachedPage> pages; // stores from segment 1 to segment n (0 is not stored here)
+
+ private int startPage;
+ private int numberOfRemainingPages;
+ private BitSet pageZeroSegmentsPages;
+
+ public ColumnMultiPageZeroBufferProvider(IColumnReadMultiPageOp multiPageOp, LongSet pinnedPages) {
+ this.multiPageOp = multiPageOp;
+ this.pinnedPages = pinnedPages;
+ this.pages = new ArrayList<>();
+ }
+
+ @Override
+ public void reset(ColumnBTreeReadLeafFrame frame) throws HyracksDataException {
+ startPage = frame.getPageId() + 1;
+ numberOfRemainingPages = frame.getNumberOfPageZeroSegments() - 1; // zeroth segment is not counted
+ pageZeroSegmentsPages = frame.getPageZeroSegmentsPages();
+ if (pageZeroSegmentsPages == null) {
+ pageZeroSegmentsPages = EMPTY_SEGMENTS;
+ }
+ }
+
+ @Override
+ public void readAll(Queue<ByteBuffer> buffers) throws HyracksDataException {
+ throw new IllegalStateException("Reading all pages is not allowed for zero buffer provider.");
+ }
+
+ public int getNumberOfRemainingPages() {
+ return numberOfRemainingPages;
+ }
+
+ public void readAll(List<ByteBuffer> buffers, Int2IntMap segmentDir) throws HyracksDataException {
+ if (pageZeroSegmentsPages == EMPTY_SEGMENTS) {
+ // All the segments are expected to present in the flash cache, but still need pinning into buffer cache.
+ // will do on request basis? or prefetch all the segments?
+ return;
+ }
+ //Since all the pageSegments are pinned for calculating the lengths of the columns,
+ //read all the segments and store them in the buffers list.
+ //after ColumnRanges.reset(), unpin the segments that are not required.
+ for (int segmentIndex = 0; segmentIndex < numberOfRemainingPages; segmentIndex++) {
+ ByteBuffer buffer = read(segmentIndex);
+ segmentDir.put(segmentIndex, buffers.size());
+ buffers.add(buffer);
+ }
+ }
+
+ public ByteBuffer read(int segmentIndex) throws HyracksDataException {
+ if (segmentIndex < 0 || segmentIndex >= numberOfRemainingPages) {
+ throw new IndexOutOfBoundsException("Segment index out of bounds: " + segmentIndex);
+ }
+ ICachedPage segment = readSegment(segmentIndex);
+ return segment.getBuffer();
+ }
+
+ @Override
+ public void releaseAll() throws HyracksDataException {
+ for (ICachedPage page : pages) {
+ if (page != null) {
+ multiPageOp.unpin(page);
+ }
+ }
+ pages.clear();
+ }
+
+ public void releasePages(IntList notRequiredSegmentsIndexes) throws HyracksDataException {
+ //From the list of cached pages, remove those pages.
+ //Pages and buffers list are in sync, so we can use the same indexes.
+ Throwable th = null;
+ for (int pageIndex : notRequiredSegmentsIndexes) {
+ if (pageIndex < 0 || pageIndex >= pages.size()) {
+ throw new IndexOutOfBoundsException("Page index out of bounds: " + pageIndex);
+ }
+ try {
+ ICachedPage page = pages.get(pageIndex);
+ if (page != null) {
+ multiPageOp.unpin(page);
+ pinnedPages.remove(((CachedPage) page).getDiskPageId());
+ pages.set(pageIndex, null); // Clear the reference
+ }
+ } catch (Exception e) {
+ if (th == null) {
+ th = e;
+ } else {
+ th.addSuppressed(e);
+ }
+ }
+ }
+ if (th != null) {
+ throw HyracksDataException.create(th);
+ }
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ throw new UnsupportedOperationException("getBuffer() is not supported for multi-page zero buffer provider.");
+ }
+
+ @Override
+ public int getLength() {
+ throw new IllegalStateException("Reading all pages is not allowed for zero buffer provider.");
+ }
+
+ private ICachedPage readSegment(int segmentIndex) throws HyracksDataException {
+ // The page segments are most likely to be present in the buffer cache,
+ // as the pages are pinned when a new pageZero is accessed.
+ ICachedPage segmentPage = multiPageOp.pin(startPage + segmentIndex);
+ pages.add(segmentPage);
+ pinnedPages.add(((CachedPage) segmentPage).getDiskPageId());
+ return segmentPage;
+ }
+
+ @Override
+ public int getColumnIndex() {
+ throw new IllegalStateException("Reading all pages is not allowed for zero buffer provider.");
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnSingleBufferProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnSingleBufferProvider.java
index 3ae5c7d..cb953fa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnSingleBufferProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/ColumnSingleBufferProvider.java
@@ -37,7 +37,7 @@
}
@Override
- public void reset(ColumnBTreeReadLeafFrame frame) {
+ public void reset(ColumnBTreeReadLeafFrame frame) throws HyracksDataException {
int offset = frame.getColumnOffset(columnIndex);
this.buffer = frame.getBuffer().duplicate();
buffer.position(offset);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
index 04b4984..c8815ae 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
@@ -61,12 +62,13 @@
public class LSMColumnBTreeUtil {
- public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
- int[] btreeFields, IMetadataPageManagerFactory freePageManagerFactory, boolean updateAware, ITracer tracer,
+ public static LSMBTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory, int[] btreeFields,
+ IMetadataPageManagerFactory freePageManagerFactory, boolean updateAware, ITracer tracer,
ICompressorDecompressorFactory compressorDecompressorFactory, ITypeTraits nullTypeTraits,
INullIntrospector nullIntrospector, IColumnManagerFactory columnManagerFactory, boolean atomic,
IDiskCacheMonitoringService diskCacheService) throws HyracksDataException {
@@ -121,10 +123,10 @@
ILSMDiskComponentFactory bulkLoadComponentFactory =
new LSMColumnBTreeWithBloomFilterDiskComponentFactory(bulkloadBTreeFactory, bloomFilterFactory);
- return new LSMColumnBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
- deleteLeafFrameFactory, diskBufferCache, fileNameManager, flushComponentFactory, mergeComponentFactory,
- bulkLoadComponentFactory, bloomFilterFalsePositiveRate, typeTraits.length, cmpFactories, mergePolicy,
- opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, btreeFields, tracer,
- columnManager, atomic, diskCacheManager);
+ return new LSMColumnBTree(storageConfig, ioManager, virtualBufferCaches, interiorFrameFactory,
+ insertLeafFrameFactory, deleteLeafFrameFactory, diskBufferCache, fileNameManager, flushComponentFactory,
+ mergeComponentFactory, bulkLoadComponentFactory, bloomFilterFalsePositiveRate, typeTraits.length,
+ cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory,
+ btreeFields, tracer, columnManager, atomic, diskCacheManager);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
index 958923f..fec254f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
@@ -95,5 +95,15 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
index 43555ca..d5c7e72 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
@@ -30,6 +30,8 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.INullIntrospector;
import org.apache.hyracks.storage.am.lsm.btree.utils.LSMBTreeUtil;
@@ -105,6 +107,7 @@
@Override
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
+ NCConfig storageConfig = ((NodeControllerService) serviceCtx.getControllerService()).getConfiguration();
IIOManager ioManager = storageManager.getIoManager(serviceCtx);
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
@@ -112,9 +115,9 @@
pageWriteCallbackFactory.initialize(serviceCtx, this);
//TODO: enable updateAwareness for secondary LSMBTree indexes
boolean updateAware = false;
- return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits,
- cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
- mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+ return LSMBTreeUtil.createLSMTree(storageConfig, ioManager, vbcs, file,
+ storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields,
+ bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory, pageWriteCallbackFactory, isPrimary, filterTypeTraits, filterCmpFactories,
btreeFields, filterFields, durable, metadataPageManagerFactory, updateAware, serviceCtx.getTracer(),
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 3526d2f..a720312 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTree;
@@ -90,7 +91,7 @@
// Primary and Primary Key LSMBTree has a Bloomfilter, but Secondary one doesn't have.
private final boolean hasBloomFilter;
- public LSMBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public LSMBTree(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache diskBufferCache,
ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
@@ -101,8 +102,8 @@
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
boolean needKeyDupCheck, boolean hasBloomFilter, int[] btreeFields, int[] filterFields, boolean durable,
boolean updateAware, ITracer tracer, boolean atomic) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
+ super(storageConfig, ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
bulkLoadComponentFactory, filterFrameFactory, filterManager, filterFields, durable, filterHelper,
btreeFields, tracer, atomic);
this.insertLeafFrameFactory = insertLeafFrameFactory;
@@ -265,8 +266,8 @@
}
component = createDiskComponent(componentFactory, flushOp.getTarget(), null,
flushOp.getBloomFilterTarget(), true);
- componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, numElements, false, false,
- false, pageWriteCallbackFactory.createPageWriteCallback());
+ componentBulkLoader = component.createBulkLoader(storageConfig, operation, 1.0f, false, numElements,
+ false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
IIndexCursor scanCursor = accessor.createSearchCursor(false);
accessor.search(scanCursor, nullPred);
try {
@@ -335,8 +336,8 @@
mergedComponent = createDiskComponent(getMergeComponentFactory(), mergeOp.getTarget(), null,
mergeOp.getBloomFilterTarget(), true);
IPageWriteCallback pageWriteCallback = pageWriteCallbackFactory.createPageWriteCallback();
- componentBulkLoader = mergedComponent.createBulkLoader(operation, 1.0f, false, numElements, false,
- false, false, pageWriteCallback);
+ componentBulkLoader = mergedComponent.createBulkLoader(storageConfig, operation, 1.0f, false,
+ numElements, false, false, false, pageWriteCallback);
while (cursor.hasNext()) {
cursor.next();
ITupleReference frameTuple = cursor.getTuple();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
index 6bd236c..a23b716 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
@@ -62,30 +63,32 @@
private LSMBTreeUtil() {
}
- public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
- boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory,
+ public static LSMBTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory, boolean needKeyDupCheck,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
+ int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory,
boolean updateAware, ITracer tracer, ICompressorDecompressorFactory compressorDecompressorFactory,
boolean hasBloomFilter, ITypeTraits nullTypeTraits, INullIntrospector nullIntrospector)
throws HyracksDataException {
- return createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits, cmpFactories,
- bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
+ return createLSMTree(storageConfig, ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
+ cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck, filterTypeTraits, filterCmpFactories,
btreeFields, filterFields, durable, freePageManagerFactory, updateAware, tracer,
compressorDecompressorFactory, hasBloomFilter, nullTypeTraits, nullIntrospector, false);
}
- public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
- boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory,
+ public static LSMBTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory, boolean needKeyDupCheck,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
+ int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory,
boolean updateAware, ITracer tracer, ICompressorDecompressorFactory compressorDecompressorFactory,
boolean hasBloomFilter, ITypeTraits nullTypeTraits, INullIntrospector nullIntrospector, boolean atomic)
throws HyracksDataException {
@@ -137,7 +140,7 @@
bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, filterHelper);
}
- return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
+ return new LSMBTree(storageConfig, ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory,
filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length,
cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
index 39e5e9c..50e534a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
@@ -102,5 +102,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java
index c77d7ff..36c1435 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.common.api;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
@@ -92,11 +93,12 @@
}
@Override
- public ChainedLSMDiskComponentBulkLoader createBulkLoader(ILSMIOOperation operation, float fillFactor,
- boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
+ public ChainedLSMDiskComponentBulkLoader createBulkLoader(NCConfig storageConfig, ILSMIOOperation operation,
+ float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
boolean cleanupEmptyComponent, IPageWriteCallback callback) throws HyracksDataException {
- ChainedLSMDiskComponentBulkLoader chainedBulkLoader = super.createBulkLoader(operation, fillFactor, verifyInput,
- numElementsHint, checkIfEmptyIndex, withFilter, cleanupEmptyComponent, callback);
+ ChainedLSMDiskComponentBulkLoader chainedBulkLoader =
+ super.createBulkLoader(storageConfig, operation, fillFactor, verifyInput, numElementsHint,
+ checkIfEmptyIndex, withFilter, cleanupEmptyComponent, callback);
if (numElementsHint > 0) {
chainedBulkLoader.addBulkLoader(createBloomFilterBulkLoader(numElementsHint, callback));
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
index e41a17e..ef2061c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
@@ -21,6 +21,7 @@
import java.util.Set;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata;
@@ -124,6 +125,7 @@
* Creates a bulkloader pipeline which includes all chained operations, bulkloading individual elements of the
* component: indexes, LSM filters, Bloom filters, buddy indexes, etc.
*
+ * @param storageConfig
* @param operation
* @param fillFactor
* @param verifyInput
@@ -134,9 +136,9 @@
* @return the created disk component bulk loader
* @throws HyracksDataException
*/
- ILSMDiskComponentBulkLoader createBulkLoader(ILSMIOOperation operation, float fillFactor, boolean verifyInput,
- long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, boolean cleanupEmptyComponent,
- IPageWriteCallback callback) throws HyracksDataException;
+ ILSMDiskComponentBulkLoader createBulkLoader(NCConfig storageConfig, ILSMIOOperation operation, float fillFactor,
+ boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
+ boolean cleanupEmptyComponent, IPageWriteCallback callback) throws HyracksDataException;
/**
* Returns all pages of the component to the buffer cache
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
index b6066fc..ea10c9a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
@@ -237,8 +238,8 @@
}
@Override
- public ChainedLSMDiskComponentBulkLoader createBulkLoader(ILSMIOOperation operation, float fillFactor,
- boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
+ public ChainedLSMDiskComponentBulkLoader createBulkLoader(NCConfig storageConfig, ILSMIOOperation operation,
+ float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
boolean cleanupEmptyComponent, IPageWriteCallback callback) throws HyracksDataException {
ChainedLSMDiskComponentBulkLoader chainedBulkLoader =
new ChainedLSMDiskComponentBulkLoader(operation, this, cleanupEmptyComponent);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 3f60978..8b16e8c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -40,6 +40,7 @@
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
@@ -109,6 +110,7 @@
protected final AtomicBoolean[] flushRequests;
protected volatile boolean memoryComponentsAllocated = false;
protected ITracer tracer;
+ protected NCConfig storageConfig;
// Factory for creating on-disk index components during flush and merge.
protected final ILSMDiskComponentFactory componentFactory;
// Factory for creating on-disk index components during bulkload.
@@ -120,7 +122,7 @@
private final ILSMMergePolicy mergePolicy;
private final ILSMIOOperationScheduler ioScheduler;
- public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public AbstractLSMIndex(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
@@ -150,6 +152,7 @@
this.temporaryDiskComponents = new ArrayList<>();
this.mergePolicy = mergePolicy;
this.ioScheduler = ioScheduler;
+ this.storageConfig = storageConfig;
fileManager.initLastUsedSeq(ioOpCallback.getLastValidSequence());
lsmHarness = new LSMHarness(this, ioScheduler, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled(),
@@ -164,7 +167,7 @@
}
}
- public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public AbstractLSMIndex(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
@@ -172,8 +175,8 @@
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
int[] filterFields, boolean durable, IComponentFilterHelper filterHelper, int[] treeFields, ITracer tracer)
throws HyracksDataException {
- this(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
+ this(storageConfig, ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
bulkLoadComponentFactory, filterFrameFactory, filterManager, filterFields, durable, filterHelper,
treeFields, tracer, false);
}
@@ -568,7 +571,7 @@
componentFileRefs.getBloomFilterFileReference(), true));
ioOpCallback.scheduled(loadOp);
opCtx.setIoOperation(loadOp);
- return new LSMIndexDiskComponentBulkLoader(this, opCtx, fillLevel, verifyInput, numElementsHint);
+ return new LSMIndexDiskComponentBulkLoader(storageConfig, this, opCtx, fillLevel, verifyInput, numElementsHint);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/EmptyComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/EmptyComponent.java
index 0dfb12e..f5a7ed1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/EmptyComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/EmptyComponent.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
@@ -137,8 +138,8 @@
}
@Override
- public ChainedLSMDiskComponentBulkLoader createBulkLoader(ILSMIOOperation operation, float fillFactor,
- boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
+ public ChainedLSMDiskComponentBulkLoader createBulkLoader(NCConfig storageConfig, ILSMIOOperation operation,
+ float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
boolean cleanupEmptyComponent, IPageWriteCallback callback) throws HyracksDataException {
return null;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java
index 88fa105..841d389 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
@@ -34,12 +35,13 @@
private final ILSMIndexOperationContext opCtx;
private boolean failed = false;
- public LSMIndexDiskComponentBulkLoader(AbstractLSMIndex lsmIndex, ILSMIndexOperationContext opCtx, float fillFactor,
- boolean verifyInput, long numElementsHint) throws HyracksDataException {
+ public LSMIndexDiskComponentBulkLoader(NCConfig storageConfig, AbstractLSMIndex lsmIndex,
+ ILSMIndexOperationContext opCtx, float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
this.lsmIndex = lsmIndex;
this.opCtx = opCtx;
- this.componentBulkLoader = opCtx.getIoOperation().getNewComponent().createBulkLoader(opCtx.getIoOperation(),
- fillFactor, verifyInput, numElementsHint, false, true, true,
+ this.componentBulkLoader = opCtx.getIoOperation().getNewComponent().createBulkLoader(storageConfig,
+ opCtx.getIoOperation(), fillFactor, verifyInput, numElementsHint, false, true, true,
lsmIndex.getPageWriteCallbackFactory().createPageWriteCallback());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
index cf10370..68608cc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
@@ -96,14 +96,23 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
index 8048a93..f20c716 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
@@ -30,6 +30,8 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.INullIntrospector;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -130,6 +132,7 @@
@Override
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
IIOManager ioManager = storageManager.getIoManager(serviceCtx);
+ NCConfig storageConfig = ((NodeControllerService) serviceCtx.getControllerService()).getConfiguration();
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
IBufferCache bufferCache = storageManager.getBufferCache(serviceCtx);
@@ -138,21 +141,22 @@
ioOpCallbackFactory.initialize(serviceCtx, this);
pageWriteCallbackFactory.initialize(serviceCtx, this);
if (isPartitioned) {
- return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits,
+ return InvertedIndexUtils.createPartitionedLSMInvertedIndex(storageConfig, ioManager, virtualBufferCaches,
+ typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ fullTextConfigEvaluatorFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate,
+ mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler,
+ ioOpCallbackFactory, pageWriteCallbackFactory, invertedIndexFields, filterTypeTraits,
+ filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
+ invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory, serviceCtx.getTracer(),
+ nullTypeTraits, nullIntrospector);
+ } else {
+ return InvertedIndexUtils.createLSMInvertedIndex(storageConfig, ioManager, virtualBufferCaches, typeTraits,
cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, fullTextConfigEvaluatorFactory,
bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory,
pageWriteCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
metadataPageManagerFactory, serviceCtx.getTracer(), nullTypeTraits, nullIntrospector);
- } else {
- return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories,
- tokenTypeTraits, tokenCmpFactories, tokenizerFactory, fullTextConfigEvaluatorFactory, bufferCache,
- file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy,
- opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory,
- pageWriteCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
- filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
- metadataPageManagerFactory, serviceCtx.getTracer(), nullTypeTraits, nullIntrospector);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index f98a056..75743c5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.PermutingTupleReference;
@@ -99,7 +100,7 @@
protected final ITypeTraits nullTypeTraits;
protected final INullIntrospector nullIntrospector;
- public LSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public LSMInvertedIndex(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ILSMDiskComponentFactory componentFactory, IComponentFilterHelper filterHelper,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
double bloomFilterFalsePositiveRate, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
@@ -111,8 +112,8 @@
int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer, ITypeTraits nullTypeTraits,
INullIntrospector nullIntrospector) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
+ super(storageConfig, ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
componentFactory, filterFrameFactory, filterManager, filterFields, durable, filterHelper,
invertedIndexFields, tracer);
this.tokenizerFactory = tokenizerFactory;
@@ -283,8 +284,8 @@
btreeCountingCursor.destroy();
}
- ILSMDiskComponentBulkLoader componentBulkLoader = component.createBulkLoader(operation, 1.0f, false,
- numBTreeTuples, false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
+ ILSMDiskComponentBulkLoader componentBulkLoader = component.createBulkLoader(storageConfig, operation, 1.0f,
+ false, numBTreeTuples, false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
// Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor(false);
@@ -367,15 +368,15 @@
numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i))
.getBloomFilter().getNumElements();
}
- componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, numElements, false, false,
- false, pageWriteCallbackFactory.createPageWriteCallback());
+ componentBulkLoader = component.createBulkLoader(storageConfig, operation, 1.0f, false, numElements,
+ false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader);
} finally {
btreeCursor.destroy();
}
} else {
- componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, 0L, false, false, false,
- pageWriteCallbackFactory.createPageWriteCallback());
+ componentBulkLoader = component.createBulkLoader(storageConfig, operation, 1.0f, false, 0L, false,
+ false, false, pageWriteCallbackFactory.createPageWriteCallback());
}
search(opCtx, cursor, mergePred);
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index a801146..61783d7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.common.api.INullIntrospector;
import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
@@ -47,24 +48,25 @@
public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
- public PartitionedLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- ILSMDiskComponentFactory componentFactory, IComponentFilterHelper filterHelper,
- ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- double bloomFilterFalsePositiveRate, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
- ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
- IBinaryTokenizerFactory tokenizerFactory, IFullTextConfigEvaluatorFactory fullTextConfigEvaluatorFactory,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ public PartitionedLSMInvertedIndex(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, ILSMDiskComponentFactory componentFactory,
+ IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory,
+ LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate, IBufferCache diskBufferCache,
+ ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
+ IFullTextConfigEvaluatorFactory fullTextConfigEvaluatorFactory, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer, ITypeTraits nullTypeTraits,
INullIntrospector nullIntrospector) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory, filterManager,
- bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, tokenizerFactory, fullTextConfigEvaluatorFactory, mergePolicy,
- opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, invertedIndexFields,
- filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, tracer,
- nullTypeTraits, nullIntrospector);
+ super(storageConfig, ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory,
+ filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ fullTextConfigEvaluatorFactory, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
+ pageWriteCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
+ invertedIndexFieldsForNonBulkLoadOps, durable, tracer, nullTypeTraits, nullIntrospector);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index aefb461..4dc3507 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
@@ -144,7 +145,7 @@
leafFrameFactory, invListCmpFactories, invListCmpFactories.length);
}
- public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager,
+ public static LSMInvertedIndex createLSMInvertedIndex(NCConfig storageConfig, IIOManager ioManager,
List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
@@ -189,16 +190,16 @@
ILSMDiskComponentFactory componentFactory = new LSMInvertedIndexDiskComponentFactory(invIndexFactory,
deletedKeysBTreeFactory, bloomFilterFactory, filterHelper);
- return new LSMInvertedIndex(ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory,
- filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits,
- invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ return new LSMInvertedIndex(storageConfig, ioManager, virtualBufferCaches, componentFactory, filterHelper,
+ filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager,
+ invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
fullTextConfigEvaluatorFactory, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
pageWriteCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable, tracer, nullTypeTraits, nullIntrospector);
}
- public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager,
- List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits,
+ public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(NCConfig storageConfig,
+ IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
IFullTextConfigEvaluatorFactory fullTextConfigEvaluatorFactory, IBufferCache diskBufferCache,
@@ -242,12 +243,13 @@
ILSMDiskComponentFactory componentFactory = new LSMInvertedIndexDiskComponentFactory(invIndexFactory,
deletedKeysBTreeFactory, bloomFilterFactory, filterHelper);
- return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, componentFactory, filterHelper,
- filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager,
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
- fullTextConfigEvaluatorFactory, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
- pageWriteCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable, tracer, nullTypeTraits, nullIntrospector);
+ return new PartitionedLSMInvertedIndex(storageConfig, ioManager, virtualBufferCaches, componentFactory,
+ filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache,
+ fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
+ tokenizerFactory, fullTextConfigEvaluatorFactory, mergePolicy, opTracker, ioScheduler,
+ ioOpCallbackFactory, pageWriteCallbackFactory, invertedIndexFields, filterFields,
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, tracer, nullTypeTraits,
+ nullIntrospector);
}
public static boolean checkTypeTraitsAllFixed(ITypeTraits[] typeTraits) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
index 84040ed..c7fbc51 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
@@ -88,6 +88,16 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
index 3a89238..bf347ba 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
@@ -31,6 +31,8 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.INullIntrospector;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -109,11 +111,12 @@
@Override
public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException {
IIOManager ioManager = storageManager.getIoManager(ncServiceCtx);
+ NCConfig storageConfig = ((NodeControllerService) ncServiceCtx.getControllerService()).getConfiguration();
FileReference fileRef = ioManager.resolve(path);
List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(ncServiceCtx, fileRef);
ioOpCallbackFactory.initialize(ncServiceCtx, this);
pageWriteCallbackFactory.initialize(ncServiceCtx, this);
- return LSMRTreeUtils.createLSMTree(ioManager, virtualBufferCaches, fileRef,
+ return LSMRTreeUtils.createLSMTree(storageConfig, ioManager, virtualBufferCaches, fileRef,
storageManager.getBufferCache(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx),
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
index e359661..c63edd7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
@@ -31,6 +31,8 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.INullIntrospector;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -101,11 +103,12 @@
@Override
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
IIOManager ioManager = storageManager.getIoManager(serviceCtx);
+ NCConfig storageConfig = ((NodeControllerService) serviceCtx.getControllerService()).getConfiguration();
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
ioOpCallbackFactory.initialize(serviceCtx, this);
pageWriteCallbackFactory.initialize(serviceCtx, this);
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, file,
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(storageConfig, ioManager, virtualBufferCaches, file,
storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index e95ef67..34d2ed8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IExtendedModificationOperationCallback;
@@ -75,7 +76,7 @@
protected final ITreeIndexFrameFactory rtreeLeafFrameFactory;
protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
- public AbstractLSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public AbstractLSMRTree(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
RTreeFrameFactory rtreeInteriorFrameFactory, RTreeFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
@@ -87,8 +88,8 @@
ILSMPageWriteCallbackFactory pageWriteCallbackFactory, IComponentFilterHelper filterHelper,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
+ super(storageConfig, ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
bulkLoadComponentFactory, filterFrameFactory, filterManager, filterFields, durable, filterHelper,
rtreeFields, ITracer.NONE);
int i = 0;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 8b88e7e..cafab7a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
@@ -72,7 +73,7 @@
public class LSMRTree extends AbstractLSMRTree {
protected final int[] buddyBTreeFields;
- public LSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public LSMRTree(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
RTreeFrameFactory rtreeInteriorFrameFactory, RTreeFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
IBufferCache diskBufferCache, ILSMIndexFileManager fileNameManager,
@@ -85,7 +86,7 @@
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
int[] rtreeFields, int[] buddyBTreeFields, int[] filterFields, boolean durable, boolean isPointMBR)
throws HyracksDataException {
- super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ super(storageConfig, ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory,
componentFactory, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
linearizerArray, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
@@ -117,8 +118,9 @@
rTreeTupleSorter.sort();
component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBTreeTarget(),
flushOp.getBloomFilterTarget(), true);
- componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, numBTreeTuples.longValue(),
- false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
+ componentBulkLoader =
+ component.createBulkLoader(storageConfig, operation, 1.0f, false, numBTreeTuples.longValue(),
+ false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
flushLoadRTree(isEmpty, rTreeTupleSorter, componentBulkLoader);
// scan the memory BTree and bulk load delete tuples
flushLoadBtree(memBTreeAccessor, componentBulkLoader, btreeNullPredicate);
@@ -319,13 +321,13 @@
numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
.getNumElements();
}
- componentBulkLoader = mergedComponent.createBulkLoader(mergeOp, 1.0f, false, numElements, false, false,
- false, pageWriteCallbackFactory.createPageWriteCallback());
+ componentBulkLoader = mergedComponent.createBulkLoader(storageConfig, mergeOp, 1.0f, false, numElements,
+ false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
mergeLoadBTree(mergeOp, opCtx, rtreeSearchPred, componentBulkLoader);
} else {
//no buddy-btree needed
- componentBulkLoader = mergedComponent.createBulkLoader(mergeOp, 1.0f, false, 0L, false, false, false,
- pageWriteCallbackFactory.createPageWriteCallback());
+ componentBulkLoader = mergedComponent.createBulkLoader(storageConfig, mergeOp, 1.0f, false, 0L, false,
+ false, false, pageWriteCallbackFactory.createPageWriteCallback());
}
//search old rtree components
while (cursor.hasNext()) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 6968b3c..2c223e4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
@@ -71,10 +72,11 @@
public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
private static final ICursorFactory cursorFactory = opCtx -> new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx);
- public LSMRTreeWithAntiMatterTuples(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- RTreeFrameFactory rtreeInteriorFrameFactory, RTreeFrameFactory rtreeLeafFrameFactory,
- ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
+ public LSMRTreeWithAntiMatterTuples(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, RTreeFrameFactory rtreeInteriorFrameFactory,
+ RTreeFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
+ ITreeIndexFrameFactory btreeLeafFrameFactory, IBufferCache diskBufferCache,
+ ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
ILSMDiskComponentFactory bulkLoadComponentFactory, IComponentFilterHelper filterHelper,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, int fieldCount,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeComparatorFactories,
@@ -82,7 +84,7 @@
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ super(storageConfig, ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache, fileManager, componentFactory,
bulkLoadComponentFactory, fieldCount, rtreeCmpFactories, btreeComparatorFactories, linearizer,
comparatorFields, linearizerArray, 0, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
@@ -113,8 +115,8 @@
try {
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
- componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, 0L, false, false,
- false, pageWriteCallbackFactory.createPageWriteCallback());
+ componentBulkLoader = component.createBulkLoader(storageConfig, operation, 1.0f, false, 0L,
+ false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
// Since the LSM-RTree is used as a secondary assumption, the
// primary key will be the last comparator in the BTree comparators
rTreeTupleSorter =
@@ -244,8 +246,8 @@
// Bulk load the tuples from all on-disk RTrees into the new RTree.
ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
- ILSMDiskComponentBulkLoader componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, 0L, false,
- false, false, pageWriteCallbackFactory.createPageWriteCallback());
+ ILSMDiskComponentBulkLoader componentBulkLoader = component.createBulkLoader(storageConfig, operation, 1.0f,
+ false, 0L, false, false, false, pageWriteCallbackFactory.createPageWriteCallback());
try {
try {
while (cursor.hasNext()) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 4b16f4a..bc5bb45 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.data.std.primitive.DoublePointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
@@ -75,17 +76,17 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
public class LSMRTreeUtils {
- public static LSMRTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILSMPageWriteCallbackFactory pageWriteCallbackFactory, ILinearizeComparatorFactory linearizeCmpFactory,
- int[] rtreeFields, int[] buddyBTreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR,
- IMetadataPageManagerFactory freePageManagerFactory, ITypeTraits nullTypeTraits,
- INullIntrospector nullIntrospector) throws HyracksDataException {
+ public static LSMRTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] buddyBTreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ boolean durable, boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory,
+ ITypeTraits nullTypeTraits, INullIntrospector nullIntrospector) throws HyracksDataException {
int valueFieldCount = buddyBTreeFields.length;
int keyFieldCount = typeTraits.length - valueFieldCount;
ITypeTraits[] btreeTypeTraits = new ITypeTraits[valueFieldCount];
@@ -138,17 +139,18 @@
ILSMDiskComponentFactory componentFactory =
new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, filterHelper);
- return new LSMRTree(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
- btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory,
- filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length,
- rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray,
- mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, rtreeFields,
- buddyBTreeFields, filterFields, durable, isPointMBR);
+ return new LSMRTree(storageConfig, ioManager, virtualBufferCaches, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache,
+ fileNameManager, componentFactory, filterHelper, filterFrameFactory, filterManager,
+ bloomFilterFalsePositiveRate, typeTraits.length, rtreeCmpFactories, btreeCmpFactories,
+ linearizeCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler,
+ ioOpCallbackFactory, pageWriteCallbackFactory, rtreeFields, buddyBTreeFields, filterFields, durable,
+ isPointMBR);
}
- public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IIOManager ioManager,
- List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
- ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
+ public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(NCConfig storageConfig,
+ IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
+ IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
@@ -237,12 +239,12 @@
ILSMDiskComponentFactory bulkLoadComponentFactory =
new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory, filterHelper);
- return new LSMRTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory,
- rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache,
- fileNameManager, componentFactory, bulkLoadComponentFactory, filterHelper, filterFrameFactory,
- filterManager, typeTraits.length, rtreeCmpFactories, btreeComparatorFactories, linearizerCmpFactory,
- comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
- pageWriteCallbackFactory, rtreeFields, filterFields, durable, isPointMBR);
+ return new LSMRTreeWithAntiMatterTuples(storageConfig, ioManager, virtualBufferCaches,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory, filterHelper,
+ filterFrameFactory, filterManager, typeTraits.length, rtreeCmpFactories, btreeComparatorFactories,
+ linearizerCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler,
+ ioOpCallbackFactory, pageWriteCallbackFactory, rtreeFields, filterFields, durable, isPointMBR);
}
public static ILinearizeComparatorFactory proposeBestLinearizer(ITypeTraits[] typeTraits, int numKeyFields)
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index a77ccdf..8169fb9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -500,7 +500,8 @@
for (ICachedPageInternal internalPage : cachedPages) {
CachedPage c = (CachedPage) internalPage;
if (c != null) {
- if (c.confiscated() || c.latch.getReadLockCount() != 0 || c.latch.getWriteHoldCount() != 0) {
+ if (c.confiscated() || c.latch.getReadLockCount() != 0 || c.latch.getWriteHoldCount() != 0
+ || c.pinCount.get() != 0) {
return false;
}
if (c.valid) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
index 88dfaef..6eff94f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
@@ -226,7 +226,8 @@
@Override
public String toString() {
- return "CachedPage:[page:" + BufferedFileHandle.getPageId(dpid) + ", compressedPageOffset:" + compressedOffset
+ return "CachedPage:[dpid: " + dpid + ", fileId: " + BufferedFileHandle.getFileId(dpid) + ", page:"
+ + BufferedFileHandle.getPageId(dpid) + ", compressedPageOffset:" + compressedOffset
+ ", compressedSize:" + compressedSize + "]";
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java
index 7b0abc2..f6d9e13 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java
@@ -35,22 +35,26 @@
import org.apache.hyracks.api.messages.IMessageBroker;
import org.apache.hyracks.api.resources.memory.IMemoryManager;
import org.apache.hyracks.api.service.IControllerService;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.util.trace.ITracer;
public class TestNCServiceContext implements INCServiceContext {
private final ILifeCycleComponentManager lccm;
private final IIOManager ioManager;
private final String nodeId;
+ private NodeControllerService ncs;
private Serializable distributedState;
private Object appCtx;
private final IMemoryManager mm;
- public TestNCServiceContext(IIOManager ioManager, String nodeId) {
+ public TestNCServiceContext(IIOManager ioManager, String nodeId) throws Exception {
this.lccm = new LifeCycleComponentManager();
this.ioManager = ioManager;
this.nodeId = nodeId;
+ this.ncs = new NodeControllerService(new NCConfig(nodeId));
mm = new IMemoryManager() {
@Override
public long getMaximumMemory() {
@@ -138,7 +142,7 @@
@Override
public IControllerService getControllerService() {
- return null;
+ return ncs;
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
index 2348a1a..d5a9a43 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
@@ -73,12 +73,12 @@
public static IHyracksTaskContext create(int frameSize, IOManager ioManager) {
try {
- INCServiceContext serviceCtx = new TestNCServiceContext(ioManager, null);
+ INCServiceContext serviceCtx = new TestNCServiceContext(ioManager, "asterix_nc1");
TestJobletContext jobletCtx = new TestJobletContext(frameSize, serviceCtx, new JobId(0));
TaskAttemptId tid = new TaskAttemptId(new TaskId(new ActivityId(new OperatorDescriptorId(0), 0), 0), 0);
IHyracksTaskContext taskCtx = new TestTaskContext(jobletCtx, tid, 1);
return taskCtx;
- } catch (HyracksException e) {
+ } catch (Exception e) {
throw new RuntimeException(e);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
index 1d16c62..788ccf3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
@@ -116,6 +116,12 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index 1477535..32a42bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -53,10 +53,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
index 8608cac..6ab4f06 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
@@ -78,10 +78,10 @@
private OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
ILSMIOOperationScheduler scheduler, ILSMIOOperationCallbackFactory ioCallbackFactory) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- scheduler, ioCallbackFactory, harness.getPageWriteCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), scheduler, ioCallbackFactory, harness.getPageWriteCallbackFactory(),
harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index fb25ee7..73f8090 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -53,10 +53,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index 3313443..9019795 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -58,9 +58,9 @@
IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields)
throws HyracksDataException {
- return LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, cmpFactories,
- bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ return LSMBTreeUtil.createLSMTree(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), typeTraits,
+ cmpFactories, bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), true, filterTypeTraits, filterCmpFactories, btreeFields,
filterFields, true, harness.getMetadataPageManagerFactory(), false, ITracer.NONE,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
index c607cae..3b6111a 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
@@ -54,9 +54,9 @@
@Test
public void deleteOrphanedFilesTest() throws Exception {
ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE };
- LSMBTreeTestContext ctx = LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, 1,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
+ LSMBTreeTestContext ctx = LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ 1, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
index bf76e20..1cfc712 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
@@ -52,10 +52,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index dace731..a07a11d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -53,10 +53,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
index d131ccb..967eca9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
@@ -54,10 +54,10 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, fieldSerdes.length,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ testCtx = LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ fieldSerdes.length, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
index = testCtx.getIndex();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
index a720ec8..8b66b82 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
@@ -100,11 +100,11 @@
protected LSMBTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- scheduler, harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory(), filtered, true, false,
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), scheduler, harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index b21a8b9..1cbc3cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -52,10 +52,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index 2476761..d902dfe 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -47,8 +47,9 @@
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes),
+ index = LSMBTreeUtil.createLSMTree(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(),
+ SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null, null), harness.getIOScheduler(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index d131b1f..9850ab3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -54,10 +54,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
index a14d4ec..44f7d6e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
@@ -110,11 +110,11 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(), pageWriteCallbackFactory,
- harness.getMetadataPageManagerFactory(), false, true, false,
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ pageWriteCallbackFactory, harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
index 4f2b2f7..4628f3a 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
@@ -76,10 +76,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index 6dec9f4..9de96db 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -56,8 +56,9 @@
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes),
+ index = LSMBTreeUtil.createLSMTree(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(),
+ SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null, null), harness.getIOScheduler(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
index 0125318..d976243 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
@@ -119,10 +119,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws HyracksDataException {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, !hasOnlyKeys,
hasOnlyKeys, harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java
index 0a4b37b..ee2b689 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java
@@ -69,8 +69,9 @@
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes),
+ index = LSMBTreeUtil.createLSMTree(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(),
+ SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null, null), harness.getIOScheduler(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index de2dfa5..58b235e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -53,10 +53,10 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ return LSMBTreeTestContext.create(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
harness.getCompressorDecompressorFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java
index 8107827..695395f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -79,7 +80,7 @@
private volatile int numFinishedMerges;
private volatile int numStartedMerges;
- public TestLsmBtree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ public TestLsmBtree(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache diskBufferCache,
ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
@@ -91,11 +92,11 @@
ILSMPageWriteCallbackFactory pageWriteCallbackFactory, boolean needKeyDupCheck, boolean hasBloomFilter,
int[] btreeFields, int[] filterFields, boolean durable, boolean updateAware, ITracer tracer)
throws HyracksDataException {
- super(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
- diskBufferCache, fileManager, componentFactory, bulkLoadComponentFactory, filterHelper,
- filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fieldCount, cmpFactories, mergePolicy,
- opTracker, ioScheduler, ioOperationCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck,
- hasBloomFilter, btreeFields, filterFields, durable, updateAware, tracer, false);
+ super(storageConfig, ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
+ deleteLeafFrameFactory, diskBufferCache, fileManager, componentFactory, bulkLoadComponentFactory,
+ filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fieldCount, cmpFactories,
+ mergePolicy, opTracker, ioScheduler, ioOperationCallbackFactory, pageWriteCallbackFactory,
+ needKeyDupCheck, hasBloomFilter, btreeFields, filterFields, durable, updateAware, tracer, false);
addModifyCallback(AllowTestOpCallback.INSTANCE);
addSearchCallback(AllowTestOpCallback.INSTANCE);
addFlushCallback(AllowTestOpCallback.INSTANCE);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeLocalResource.java
index 04fa11f..5928d5b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeLocalResource.java
@@ -29,6 +29,8 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResource;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -76,6 +78,7 @@
@Override
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
IIOManager ioManager = storageManager.getIoManager(serviceCtx);
+ NCConfig storageConfig = ((NodeControllerService) serviceCtx.getControllerService()).getConfiguration();
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
for (int i = 0; i < vbcs.size(); i++) {
@@ -87,9 +90,9 @@
}
ioOpCallbackFactory.initialize(serviceCtx, this);
pageWriteCallbackFactory.initialize(serviceCtx, this);
- return TestLsmBtreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx),
- typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
- mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+ return TestLsmBtreeUtil.createLSMTree(storageConfig, ioManager, vbcs, file,
+ storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields,
+ bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory, pageWriteCallbackFactory, isPrimary, filterTypeTraits, filterCmpFactories,
btreeFields, filterFields, durable, metadataPageManagerFactory, false, serviceCtx.getTracer());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java
index dd42838..3fa5745 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
@@ -59,13 +60,14 @@
private TestLsmBtreeUtil() {
}
- public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
- boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory,
+ public static LSMBTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory, boolean needKeyDupCheck,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
+ int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory,
boolean updateAware, ITracer tracer) throws HyracksDataException {
LSMBTreeTupleWriterFactory insertTupleWriterFactory =
new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false, updateAware, null, null);
@@ -114,10 +116,11 @@
bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, filterHelper);
}
- return new TestLsmBtree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
- deleteLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory,
- filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length,
- cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory,
- needKeyDupCheck, hasBloomFilter, btreeFields, filterFields, durable, updateAware, tracer);
+ return new TestLsmBtree(storageConfig, ioManager, virtualBufferCaches, interiorFrameFactory,
+ insertLeafFrameFactory, deleteLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory,
+ bulkLoadComponentFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
+ typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
+ pageWriteCallbackFactory, needKeyDupCheck, hasBloomFilter, btreeFields, filterFields, durable,
+ updateAware, tracer);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index 66af4b2..e4004a9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -54,9 +54,9 @@
@Override
protected ITreeIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields) throws HyracksDataException {
- return LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, cmpFactories,
- bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ return LSMBTreeUtil.createLSMTree(harness.getNCConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), typeTraits,
+ cmpFactories, bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), true, null, null, null, null, true,
harness.getMetadataPageManagerFactory(), false, ITracer.NONE,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index 6126040..c099ad2 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -30,6 +30,7 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
import org.apache.hyracks.storage.am.common.datagen.TupleBatch;
@@ -66,6 +67,7 @@
protected IHyracksTaskContext ctx;
protected IOManager ioManager;
protected int ioDeviceId;
+ protected NCConfig ncConfig;
protected IBufferCache bufferCache;
protected int lsmtreeFileId;
@@ -95,6 +97,7 @@
TestStorageManagerComponentHolder.init(this.onDiskPageSize, this.onDiskNumPages, MAX_OPEN_FILES);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
ioManager = TestStorageManagerComponentHolder.getIOManager();
+ ncConfig = new NCConfig(null);
ioDeviceId = 0;
file = ioManager.resolveAbsolutePath(onDiskDir);
@@ -118,7 +121,7 @@
}
}, Integer.MAX_VALUE, Integer.MAX_VALUE);
- lsmtree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, typeTraits,
+ lsmtree = LSMBTreeUtil.createLSMTree(ncConfig, ioManager, virtualBufferCaches, file, bufferCache, typeTraits,
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, new NoMergePolicy(),
new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallbackFactory.INSTANCE,
NoOpPageWriteCallbackFactory.INSTANCE, true, null, null, null, null, true,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 82f6f4b..8f60c1f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.utils.SerdeUtils;
import org.apache.hyracks.storage.am.btree.OrderedIndexTestContext;
import org.apache.hyracks.storage.am.common.CheckTuple;
@@ -76,11 +77,11 @@
upsertCheckTuple(checkTuple, checkTuples);
}
- public static LSMBTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ISerializerDeserializer[] fieldSerdes, int numKeyFields,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ public static LSMBTreeTestContext create(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ISerializerDeserializer[] fieldSerdes, int numKeyFields, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory, boolean filtered, boolean needKeyDupCheck,
boolean updateAware, ICompressorDecompressorFactory compressorDecompressorFactory)
throws HyracksDataException {
@@ -100,16 +101,16 @@
}
int[] filterfields = { btreefields.length };
IBinaryComparatorFactory[] filterCmp = { cmpFactories[0] };
- lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
- cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck, filterTypeTraits,
- filterCmp, btreefields, filterfields, true, metadataPageManagerFactory, updateAware, ITracer.NONE,
- compressorDecompressorFactory, true, null, null);
+ lsmTree = LSMBTreeUtil.createLSMTree(storageConfig, ioManager, virtualBufferCaches, file, diskBufferCache,
+ typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck,
+ filterTypeTraits, filterCmp, btreefields, filterfields, true, metadataPageManagerFactory,
+ updateAware, ITracer.NONE, compressorDecompressorFactory, true, null, null);
} else {
- lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
- cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck, null, null, null, null,
- true, metadataPageManagerFactory,
+ lsmTree = LSMBTreeUtil.createLSMTree(storageConfig, ioManager, virtualBufferCaches, file, diskBufferCache,
+ typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck, null, null,
+ null, null, true, metadataPageManagerFactory,
updateAware, new Tracer(LSMBTreeTestContext.class.getSimpleName(),
ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry()),
compressorDecompressorFactory, true, null, null);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index 4d2b086..3eb9bf0 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -32,6 +32,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
@@ -76,6 +77,7 @@
private final ICompressorDecompressorFactory compressorDecompressorFactory;
protected IOManager ioManager;
+ protected NCConfig ncConfig;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
protected List<IVirtualBufferCache> virtualBufferCaches;
@@ -113,6 +115,7 @@
this.ioOpCallbackFactory = new CountingIoOperationCallbackFactory();
this.pageWriteCallbackFactory = NoOpPageWriteCallbackFactory.INSTANCE;
this.compressorDecompressorFactory = compressorDecompressorFactory;
+ this.ncConfig = new NCConfig(null);
}
public void setUp() throws HyracksDataException {
@@ -181,6 +184,10 @@
return ioManager;
}
+ public NCConfig getNCConfig() {
+ return ncConfig;
+ }
+
public int getIODeviceId() {
return ioDeviceId;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
index 9e2cd3d..6d17134 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
@@ -106,6 +106,12 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
index c220697..0960608 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.freepage.AppendOnlyLinkedMetadataPageManagerFactory;
@@ -67,6 +68,7 @@
protected final int numMutableComponents;
protected IOManager ioManager;
+ protected NCConfig ncConfig;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
protected List<IVirtualBufferCache> virtualBufferCaches;
@@ -101,6 +103,7 @@
this.ioOpCallbackFactory = NoOpIOOperationCallbackFactory.INSTANCE;
this.pageWriteCallbackFactory = NoOpPageWriteCallbackFactory.INSTANCE;
this.numMutableComponents = AccessMethodTestsConfig.LSM_INVINDEX_NUM_MUTABLE_COMPONENTS;
+ this.ncConfig = new NCConfig(null);
}
public void setUp() throws HyracksDataException {
@@ -177,6 +180,10 @@
return ioManager;
}
+ public NCConfig getNCConfig() {
+ return ncConfig;
+ }
+
public int getIODeviceId() {
return ioDeviceId;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index e745f5d..b5da79f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -34,6 +34,7 @@
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.utils.SerdeUtils;
@@ -134,6 +135,7 @@
throws HyracksDataException {
ITypeTraits[] allTypeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IOManager ioManager = harness.getIOManager();
+ NCConfig storageConfig = harness.getNCConfig();
IBinaryComparatorFactory[] allCmpFactories =
SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
// Set token type traits and comparators.
@@ -186,8 +188,8 @@
break;
}
case LSM: {
- invIndex =
- InvertedIndexUtils.createLSMInvertedIndex(ioManager, harness.getVirtualBufferCaches(),
+ invIndex = InvertedIndexUtils
+ .createLSMInvertedIndex(storageConfig, ioManager, harness.getVirtualBufferCaches(),
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
tokenizerFactory, fullTextConfigEvaluatorFactory, harness.getDiskBufferCache(),
harness.getOnDiskDir(), harness.getBoomFilterFalsePositiveRate(),
@@ -203,7 +205,7 @@
}
case PARTITIONED_LSM: {
invIndex = InvertedIndexUtils
- .createPartitionedLSMInvertedIndex(ioManager, harness.getVirtualBufferCaches(),
+ .createPartitionedLSMInvertedIndex(storageConfig, ioManager, harness.getVirtualBufferCaches(),
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
tokenizerFactory, fullTextConfigEvaluatorFactory, harness.getDiskBufferCache(),
harness.getOnDiskDir(), harness.getBoomFilterFalsePositiveRate(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
index fca340e..ab764be 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
@@ -94,6 +94,12 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
index 21b6a70..e642de4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
@@ -56,11 +56,12 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
+ return LSMRTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
index 0662540..dc23f3b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
@@ -56,11 +56,12 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
+ return LSMRTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index 9e6fb83..71e2eb3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -43,11 +43,12 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws HyracksDataException {
- return LSMRTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
- harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ return LSMRTreeUtils.createLSMTree(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), rtreeFields, btreeFields,
filterTypeTraits, filterCmpFactories, filterFields, true, false,
harness.getMetadataPageManagerFactory(), null, null);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
index a48bfaf..515f3de 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
@@ -56,11 +56,12 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
+ return LSMRTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
index aa2bb63..4afb98e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
@@ -66,11 +66,12 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- RTreePolicyType.RTREE, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
+ testCtx = LSMRTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, RTreePolicyType.RTREE, harness.getBoomFilterFalsePositiveRate(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory());
index = testCtx.getIndex();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
index 864d555..1bd6ce0 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
@@ -55,11 +55,12 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
+ return LSMRTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java
index dcdc8b5..134d71b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java
@@ -56,11 +56,12 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), true);
+ return LSMRTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory(), true);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
index 1ac7d0b..30606b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
@@ -56,11 +56,11 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory());
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
index 2d74823..d55514b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
@@ -57,11 +57,11 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory());
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index baf54f3..49fb157 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -43,9 +43,9 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws HyracksDataException {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), rtreeFields,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
index 4872e68..15301de 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
@@ -56,11 +56,11 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory());
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
index ccce785..5b08886 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
@@ -66,7 +66,7 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(),
+ testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getNcConfig(), harness.getIOManager(),
harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
valueProviderFactories, numKeys, RTreePolicyType.RTREE, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
index df52549..fb8f63e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
@@ -55,11 +55,11 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory());
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index 980f21f..8d0ef95 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -60,11 +60,12 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] btreeFields) throws HyracksDataException {
- return LSMRTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
- harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ return LSMRTreeUtils.createLSMTree(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null,
null, null, true, false, harness.getMetadataPageManagerFactory(), null, null);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index f40add9..dd1857f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -61,9 +61,9 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] btreeFields) throws HyracksDataException {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, null, null, null, true,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 02742a8..3111745 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.utils.SerdeUtils;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -74,24 +75,25 @@
return lsmTree.getComparatorFactories();
}
- public static LSMRTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ISerializerDeserializer[] fieldSerdes,
- IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ public static LSMRTreeTestContext create(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ int numKeyFields, RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory) throws Exception {
- return create(ioManager, virtualBufferCaches, file, diskBufferCache, fieldSerdes, valueProviderFactories,
- numKeyFields, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
- ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory, false);
+ return create(storageConfig, ioManager, virtualBufferCaches, file, diskBufferCache, fieldSerdes,
+ valueProviderFactories, numKeyFields, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory,
+ false);
}
- public static LSMRTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ISerializerDeserializer[] fieldSerdes,
- IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ public static LSMRTreeTestContext create(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ int numKeyFields, RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory, boolean isPointMBR) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories =
@@ -105,9 +107,9 @@
}
IBinaryComparatorFactory[] btreeCmpFactories =
SerdeUtils.serdesToComparatorFactories(btreeFieldSerdes, numBtreeFields);
- LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache,
- typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
+ LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(storageConfig, ioManager, virtualBufferCaches, file,
+ diskBufferCache, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
+ rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
pageWriteCallbackFactory, LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length),
null, btreeFields, null, null, null, true, isPointMBR, metadataPageManagerFactory, null, null);
LSMRTreeTestContext testCtx = new LSMRTreeTestContext(fieldSerdes, lsmTree);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
index e189898..58b9a34 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.freepage.AppendOnlyLinkedMetadataPageManagerFactory;
@@ -66,6 +67,7 @@
protected final int numMutableComponents;
protected IOManager ioManager;
+ protected NCConfig ncConfig;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
protected List<IVirtualBufferCache> virtualBufferCaches;
@@ -98,6 +100,7 @@
this.ioOpCallbackFactory = NoOpIOOperationCallbackFactory.INSTANCE;
this.pageWriteCallbackFactory = NoOpPageWriteCallbackFactory.INSTANCE;
this.numMutableComponents = AccessMethodTestsConfig.LSM_RTREE_NUM_MUTABLE_COMPONENTS;
+ this.ncConfig = new NCConfig(null);
}
public void setUp() throws HyracksDataException {
@@ -209,6 +212,10 @@
return mergePolicy;
}
+ public NCConfig getNcConfig() {
+ return ncConfig;
+ }
+
public ILSMIOOperationCallbackFactory getIOOperationCallbackFactory() {
return ioOpCallbackFactory;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
index ff953f1..45b5911 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.dataflow.common.utils.SerdeUtils;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -74,7 +75,7 @@
return lsmTree.getComparatorFactories();
}
- public static LSMRTreeWithAntiMatterTuplesTestContext create(IIOManager ioManager,
+ public static LSMRTreeWithAntiMatterTuplesTestContext create(NCConfig storageConfig, IIOManager ioManager,
List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
int numKeyFields, RTreePolicyType rtreePolicyType, ILSMMergePolicy mergePolicy,
@@ -86,7 +87,7 @@
SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
IBinaryComparatorFactory[] btreeCmpFactories =
SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
- LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager,
+ LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(storageConfig, ioManager,
virtualBufferCaches, file, diskBufferCache, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
pageWriteCallbackFactory, LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length),