[ASTERIXDB-3394][STO] Introducing size bounded concurrent merge policy
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Added storage property to set max mergable component size.
- The new merge policy will schedule merges similar to the current ConcurrentMergePolicy but it will consider the max mergable component size.
Change-Id: I0cf0cfcfe5e7ff0131430ce2039ba89929fda274
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18264
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ritik Raj <raj.ritik9835@gmail.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
index e754c77..093b4c1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
@@ -35,6 +35,7 @@
import org.apache.asterix.common.ioopcallbacks.LSMIndexIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMIndexPageWriteCallbackFactory;
import org.apache.asterix.common.library.LibraryDescriptor;
+import org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicyFactory;
import org.apache.asterix.common.transactions.Checkpoint;
import org.apache.asterix.dataflow.data.common.AListElementTokenFactory;
import org.apache.asterix.dataflow.data.common.AOrderedListBinaryTokenizerFactory;
@@ -165,6 +166,7 @@
registeredClasses.put("NoMergePolicyFactory", NoMergePolicyFactory.class);
registeredClasses.put("PrefixMergePolicyFactory", PrefixMergePolicyFactory.class);
registeredClasses.put("ConcurrentMergePolicyFactory", ConcurrentMergePolicyFactory.class);
+ registeredClasses.put("SizeBoundedConcurrentMergePolicy", SizeBoundedConcurrentMergePolicyFactory.class);
registeredClasses.put("ConstantMergePolicyFactory", ConstantMergePolicyFactory.class);
registeredClasses.put("CorrelatedPrefixMergePolicyFactory", CorrelatedPrefixMergePolicyFactory.class);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index eee344f..cd52b9d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -68,6 +68,7 @@
import org.apache.asterix.common.replication.IReplicationStrategyFactory;
import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
import org.apache.asterix.common.storage.IReplicaManager;
+import org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicyFactory;
import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.asterix.common.transactions.IRecoveryManagerFactory;
@@ -107,7 +108,6 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
-import org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.GreedyScheduler;
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.buffercache.BufferCache;
@@ -229,7 +229,7 @@
IPageReplacementStrategy prs = new ClockPageReplacementStrategy(bufferAllocator, pageAllocator,
storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
lsmIOScheduler = createIoScheduler(storageProperties);
- metadataMergePolicyFactory = new ConcurrentMergePolicyFactory();
+ metadataMergePolicyFactory = new SizeBoundedConcurrentMergePolicyFactory();
indexCheckpointManagerProvider = new IndexCheckpointManagerProvider(persistenceIOManager);
ILocalResourceRepositoryFactory persistentLocalResourceRepositoryFactory =
new PersistentLocalResourceRepositoryFactory(persistenceIOManager, indexCheckpointManagerProvider,
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.3.query.sqlpp
index 6c78eaa..2f4a774 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.3.query.sqlpp
@@ -17,6 +17,20 @@
* under the License.
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.MetatypeDataverseName,
+ l.MetatypeName,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.3.query.sqlpp
index 6c78eaa..2f4a774 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.3.query.sqlpp
@@ -17,6 +17,20 @@
* under the License.
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.MetatypeDataverseName,
+ l.MetatypeName,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.3.query.sqlpp
index ad48535..6a4771a 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.3.query.sqlpp
@@ -17,6 +17,20 @@
* under the License.
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
\ No newline at end of file
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.MetatypeDataverseName,
+ l.MetatypeName,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-4/dataset_with_meta-4.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-4/dataset_with_meta-4.3.query.sqlpp
index 6c78eaa..2f4a774 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-4/dataset_with_meta-4.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-4/dataset_with_meta-4.3.query.sqlpp
@@ -17,6 +17,20 @@
* under the License.
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.MetatypeDataverseName,
+ l.MetatypeName,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.3.query.sqlpp
index 6c78eaa..2f4a774 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.3.query.sqlpp
@@ -17,6 +17,20 @@
* under the License.
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.MetatypeDataverseName,
+ l.MetatypeName,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.3.query.sqlpp
index 9ccb9d4..c519613 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.3.query.sqlpp
@@ -23,6 +23,18 @@
* Issue : 251
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.3.query.sqlpp
index 797048f..b04a07b 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.3.query.sqlpp
@@ -23,6 +23,18 @@
* Issue : 251
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.3.query.sqlpp
index 80da3cc..e1c3544 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.3.query.sqlpp
@@ -24,6 +24,18 @@
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta02/meta02.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta02/meta02.3.query.sqlpp
index c528225..9608008 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta02/meta02.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta02/meta02.3.query.sqlpp
@@ -22,7 +22,20 @@
* Date : 15 Sep 2012
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName = 'testdv' AND ds.DatasetName = 'dst01';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.ExternalDetails,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName = 'testdv' AND l.DatasetName = 'dst01';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta09/meta09.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta09/meta09.3.query.sqlpp
index d3f038a..02b876a 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta09/meta09.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/meta09/meta09.3.query.sqlpp
@@ -22,6 +22,19 @@
* Date : Sep 17 2012
*/
-SELECT VALUE ds
-FROM Metadata.`Dataset` ds
-WHERE ds.DataverseName='test' and ds.DatasetName='t1';
\ No newline at end of file
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.ExternalDetails,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.BlockLevelStorageCompression,
+ l.DatasetFormat
+FROM Metadata.`Dataset` l
+WHERE l.DataverseName='test' and l.DatasetName='t1';
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/metadata_dataset/metadata_dataset.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/metadata_dataset/metadata_dataset.3.query.sqlpp
index 7046f2a..d0e6f9f 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/metadata_dataset/metadata_dataset.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/metadata/queries/basic/metadata_dataset/metadata_dataset.3.query.sqlpp
@@ -18,7 +18,19 @@
*/
use Metadata;
-SELECT VALUE ds
-FROM `Dataset` ds
-WHERE ds.DataverseName = 'Metadata';
+SELECT l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.ExternalDetails,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.DatasetFormat
+FROM `Dataset` l
+WHERE l.DataverseName = 'Metadata';
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm
index 96ca16e..5f20f4e 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Sun Jun 30 09:05:04 PDT 2019", "DatasetId": 102, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "AuxiliaryType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Sun Jun 30 09:05:04 PDT 2019", "DatasetId": 102, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "AuxiliaryType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm
index ce2dc7c..9db69f8 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Sun Jun 30 09:05:04 PDT 2019", "DatasetId": 103, "PendingOp": 0, "MetatypeDataverseName": "meta", "MetatypeName": "AuxiliaryType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Sun Jun 30 09:05:04 PDT 2019", "DatasetId": 103, "PendingOp": 0, "MetatypeDataverseName": "meta", "MetatypeName": "AuxiliaryType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-3/dataset_with_meta-2.3.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-3/dataset_with_meta-2.3.adm
index 71072f4..8ed807d 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-3/dataset_with_meta-2.3.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-3/dataset_with_meta-2.3.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:26 PDT 2016", "DatasetId": 103, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "LineType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:26 PDT 2016", "DatasetId": 103, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "LineType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm
index f2e0319..433a96c 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false, "KeySourceIndicator": [ 1 ] }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:26 PDT 2016", "DatasetId": 104, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "LineType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false, "KeySourceIndicator": [ 1 ] }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:26 PDT 2016", "DatasetId": 104, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "LineType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm
index 7c2d1ae..641332e 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "key" ] ], "PrimaryKey": [ [ "key" ] ], "Autogenerated": false, "KeySourceIndicator": [ 1 ] }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:26 PDT 2016", "DatasetId": 105, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "AuxiliaryType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "key" ] ], "PrimaryKey": [ [ "key" ] ], "Autogenerated": false, "KeySourceIndicator": [ 1 ] }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:26 PDT 2016", "DatasetId": 105, "PendingOp": 0, "MetatypeDataverseName": "test", "MetatypeName": "AuxiliaryType", "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.1.adm
index 792935f..1047ca5 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_2/issue_251_dataset_hint_2.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ { "Name": "CARDINALITY", "Value": "2000" } }}, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "DatasetId": 114, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ { "Name": "CARDINALITY", "Value": "2000" } }}, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "DatasetId": 114, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.1.adm
index e612895..cc18eba 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_3/issue_251_dataset_hint_3.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ { "Name": "CARDINALITY", "Value": "2000" } }}, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "DatasetId": 115, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ { "Name": "CARDINALITY", "Value": "2000" } }}, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "DatasetId": 115, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.1.adm
index ac171aa..b2358ce 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/issue_251_dataset_hint_4/issue_251_dataset_hint_4.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "DatasetId": 116, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "test.Book", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "DatasetId": 116, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta02/meta02.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta02/meta02.1.adm
index 39391ca..6af4e36 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta02/meta02.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta02/meta02.1.adm
@@ -1 +1 @@
-{ "DataverseName": "testdv", "DatasetName": "dst01", "DatatypeDataverseName": "testdv", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "testdv.dst01", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:27 PDT 2016", "DatasetId": 108, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "testdv", "DatasetName": "dst01", "DatatypeDataverseName": "testdv", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "testdv.dst01", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:27 PDT 2016", "DatasetId": 108, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta09/meta09.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta09/meta09.1.adm
index c63e98e..70f437e 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta09/meta09.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta09/meta09.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "t1", "DatatypeDataverseName": "test", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test.t1", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:27 PDT 2016", "DatasetId": 110, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "test", "DatasetName": "t1", "DatatypeDataverseName": "test", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test.t1", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 15:54:27 PDT 2016", "DatasetId": 110, "PendingOp": 0, "BlockLevelStorageCompression": { "DatasetCompressionScheme": "snappy" }, "DatasetFormat": { "Format": "ROW" } }
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_compaction_policy/metadata_compaction_policy.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_compaction_policy/metadata_compaction_policy.1.adm
index e8dbbe9..4f537a6 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_compaction_policy/metadata_compaction_policy.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_compaction_policy/metadata_compaction_policy.1.adm
@@ -3,3 +3,4 @@
{ "DataverseName": "Metadata", "CompactionPolicy": "correlated-prefix", "Classname": "org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory" }
{ "DataverseName": "Metadata", "CompactionPolicy": "no-merge", "Classname": "org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory" }
{ "DataverseName": "Metadata", "CompactionPolicy": "prefix", "Classname": "org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory" }
+{ "DataverseName": "Metadata", "CompactionPolicy": "size-bounded-concurrent", "Classname": "org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicyFactory" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
index ca99535..6b6e4dd 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
@@ -1,17 +1,17 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 13, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 2, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 8, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 3, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DatatypeDataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ] ], "PrimaryKey": [ [ "DataverseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 1, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DatatypeDataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 14, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "DatatypeDataverseName": "Metadata", "DatatypeName": "FeedRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FeedName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FeedName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 10, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "FeedConnection", "DatatypeDataverseName": "Metadata", "DatatypeName": "FeedConnectionRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FeedName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FeedName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 11, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "PolicyName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "PolicyName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 12, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "FullTextConfig", "DatatypeDataverseName": "Metadata", "DatatypeName": "FullTextConfigRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FullTextConfigName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FullTextConfigName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Oct 29 16:14:12 PDT 2020", "DatasetId": 16, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "FullTextFilter", "DatatypeDataverseName": "Metadata", "DatatypeName": "FullTextFilterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FullTextFilterName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FullTextFilterName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Oct 29 16:14:12 PDT 2020", "DatasetId": 17, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "DatatypeDataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ], [ "Arity" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ], [ "Arity" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 7, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "DatatypeDataverseName": "Metadata", "DatatypeName": "IndexRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "IndexName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "IndexName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 4, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "DatatypeDataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 9, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "DatatypeDataverseName": "Metadata", "DatatypeName": "NodeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "NodeName" ] ], "PrimaryKey": [ [ "NodeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 5, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DatatypeDataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "GroupName" ] ], "PrimaryKey": [ [ "GroupName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 6, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Synonym", "DatatypeDataverseName": "Metadata", "DatatypeName": "SynonymRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "SynonymName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "SynonymName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Dec 17 10:36:07 PST 2019", "DatasetId": 15, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 13, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 2, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 8, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 3, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DatatypeDataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ] ], "PrimaryKey": [ [ "DataverseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 1, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DatatypeDataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 14, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "DatatypeDataverseName": "Metadata", "DatatypeName": "FeedRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FeedName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FeedName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 10, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "FeedConnection", "DatatypeDataverseName": "Metadata", "DatatypeName": "FeedConnectionRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FeedName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FeedName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 11, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "PolicyName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "PolicyName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 12, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "FullTextConfig", "DatatypeDataverseName": "Metadata", "DatatypeName": "FullTextConfigRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FullTextConfigName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FullTextConfigName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Oct 29 16:14:12 PDT 2020", "DatasetId": 16, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "FullTextFilter", "DatatypeDataverseName": "Metadata", "DatatypeName": "FullTextFilterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "FullTextFilterName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "FullTextFilterName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Oct 29 16:14:12 PDT 2020", "DatasetId": 17, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "DatatypeDataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ], [ "Arity" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ], [ "Arity" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 7, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "DatatypeDataverseName": "Metadata", "DatatypeName": "IndexRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "IndexName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "IndexName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 4, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "DatatypeDataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 9, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "DatatypeDataverseName": "Metadata", "DatatypeName": "NodeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "NodeName" ] ], "PrimaryKey": [ [ "NodeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 5, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DatatypeDataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "GroupName" ] ], "PrimaryKey": [ [ "GroupName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016", "DatasetId": 6, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Synonym", "DatatypeDataverseName": "Metadata", "DatatypeName": "SynonymRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "SynonymName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "SynonymName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Dec 17 10:36:07 PST 2019", "DatasetId": 15, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.query.sqlpp
index d71e6f5..22f7d8b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.query.sqlpp
@@ -30,8 +30,6 @@
l.DatatypeName,
l.DatasetType,
l.GroupName,
- l.CompactionPolicy,
- l.CompactionPolicyProperties,
l.InternalDetails,
l.`Hints`,
l.Timestamp,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.query.sqlpp
index 5435d9e..d078f57 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.query.sqlpp
@@ -33,8 +33,6 @@
l.DatatypeName,
l.DatasetType,
l.GroupName,
- l.CompactionPolicy,
- l.CompactionPolicyProperties,
l.InternalDetails,
l.`Hints`,
l.Timestamp,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.query.sqlpp
index c7ef7cd..5def8ec 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.query.sqlpp
@@ -28,8 +28,6 @@
l.DatatypeName,
l.DatasetType,
l.GroupName,
- l.CompactionPolicy,
- l.CompactionPolicyProperties,
l.ExternalDetails,
l.InternalDetails,
l.`Hints`,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/udf23/udf23.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/udf23/udf23.3.query.sqlpp
index 91359a7..6cf3238 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/udf23/udf23.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/udf23/udf23.3.query.sqlpp
@@ -25,7 +25,20 @@
use test;
-select element l
+select l.DatabaseName,
+ l.DataverseName,
+ l.DatasetName,
+ l.DatatypeDataverseName,
+ l.DatatypeName,
+ l.DatasetType,
+ l.GroupName,
+ l.InternalDetails,
+ l.`Hints`,
+ l.Timestamp,
+ l.DatasetId,
+ l.PendingOp,
+ l.DatatypeDatabaseName,
+ l.DatasetFormat
from `Metadata`.`Dataset` as l
where (l.DataverseName = 'Metadata')
order by l.DatasetName
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv02/cross-dv02.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv02/cross-dv02.1.adm
index 57f1f41..59b046b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv02/cross-dv02.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv02/cross-dv02.1.adm
@@ -1,4 +1,4 @@
-{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.gdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 05:50:01 UTC 2019", "DatasetId": 3091, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.prof", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1861, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.pstdoc", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1862, "PendingOp": 0 }
-{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.ugdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1859, "PendingOp": 0 }
+{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.gdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 05:50:01 UTC 2019", "DatasetId": 3091, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.prof", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1861, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.pstdoc", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1862, "PendingOp": 0 }
+{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.ugdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1859, "PendingOp": 0 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv04/cross-dv04.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv04/cross-dv04.1.adm
index 57f1f41..59b046b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv04/cross-dv04.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv04/cross-dv04.1.adm
@@ -1,4 +1,4 @@
-{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.gdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 05:50:01 UTC 2019", "DatasetId": 3091, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.prof", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1861, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.pstdoc", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1862, "PendingOp": 0 }
-{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.ugdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1859, "PendingOp": 0 }
+{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.gdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 05:50:01 UTC 2019", "DatasetId": 3091, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.prof", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1861, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "teacher.pstdoc", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1862, "PendingOp": 0 }
+{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "student.ugdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Jun 21 16:06:36 PDT 2016", "DatasetId": 1859, "PendingOp": 0 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
index e8e2aac..0fd3c66 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
@@ -1,7 +1,7 @@
-{ "DataverseName": "test1", "DatasetName": "TwitterData", "DatatypeDataverseName": "test1", "DatatypeName": "Tweet", "DatasetType": "EXTERNAL", "GroupName": "test1.TwitterData", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "ExternalDetails": { "DatasourceAdapter": "localfs", "Properties": [ { "Name": "path", "Value": "asterix_nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ], "LastRefreshTime": datetime("2019-06-28T05:02:44.665Z"), "TransactionState": 0 }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 128, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t1", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test1.t1", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 122, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t2", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test1.t2", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 125, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t3", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test1.t3", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 126, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t2", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test2.t2", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 123, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t3", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test2.t3", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 124, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t4", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test2.t4", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 127, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "TwitterData", "DatatypeDataverseName": "test1", "DatatypeName": "Tweet", "DatasetType": "EXTERNAL", "GroupName": "test1.TwitterData", "ExternalDetails": { "DatasourceAdapter": "localfs", "Properties": [ { "Name": "path", "Value": "asterix_nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ], "LastRefreshTime": datetime("2019-06-28T05:02:44.665Z"), "TransactionState": 0 }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 128, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t1", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test1.t1", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 122, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t2", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test1.t2", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 125, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t3", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test1.t3", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 126, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t2", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test2.t2", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 123, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t3", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test2.t3", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 124, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t4", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "test2.t4", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Jun 27 22:02:44 PDT 2019", "DatasetId": 127, "PendingOp": 0 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
index c69e050..aa10852 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
@@ -1,6 +1,6 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 13, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 2, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 8, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 3, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DatatypeDataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ] ], "PrimaryKey": [ [ "DataverseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 1, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
-{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DatatypeDataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 14, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "CompactionPolicy" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 13, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 2, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 8, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatatypeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 3, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DatatypeDataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ] ], "PrimaryKey": [ [ "DataverseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 1, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DatatypeDataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "PrimaryKey": [ [ "DataverseName" ], [ "DatasetName" ], [ "FileNumber" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Fri May 19 12:41:05 PDT 2023", "DatasetId": 14, "PendingOp": 0, "DatasetFormat": { "Format": "ROW" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv02/cross-dv02.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv02/cross-dv02.1.adm
index 95c9207..72b4b85 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv02/cross-dv02.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv02/cross-dv02.1.adm
@@ -1,4 +1,4 @@
-{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.gdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:28 PST 2023", "DatasetId": 103, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.prof", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:28 PST 2023", "DatasetId": 104, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.pstdoc", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:28 PST 2023", "DatasetId": 105, "PendingOp": 0 }
-{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.ugdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:27 PST 2023", "DatasetId": 102, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.gdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:28 PST 2023", "DatasetId": 103, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.prof", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:28 PST 2023", "DatasetId": 104, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.pstdoc", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:28 PST 2023", "DatasetId": 105, "PendingOp": 0 }
+{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.ugdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:27 PST 2023", "DatasetId": 102, "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv04/cross-dv04.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv04/cross-dv04.1.adm
index d20b7d6..8f90b6e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv04/cross-dv04.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv04/cross-dv04.1.adm
@@ -1,4 +1,4 @@
-{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.gdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:32 PST 2023", "DatasetId": 111, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.prof", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:32 PST 2023", "DatasetId": 112, "PendingOp": 0 }
-{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.pstdoc", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:33 PST 2023", "DatasetId": 113, "PendingOp": 0 }
-{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.ugdstd", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:32 PST 2023", "DatasetId": 110, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "student", "DatasetName": "gdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.gdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:32 PST 2023", "DatasetId": 111, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "prof", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.prof", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:32 PST 2023", "DatasetId": 112, "PendingOp": 0 }
+{ "DataverseName": "teacher", "DatasetName": "pstdoc", "DatatypeDataverseName": "teacher", "DatatypeName": "tchrType", "DatasetType": "INTERNAL", "GroupName": "Default.teacher.pstdoc", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:33 PST 2023", "DatasetId": 113, "PendingOp": 0 }
+{ "DataverseName": "student", "DatasetName": "ugdstd", "DatatypeDataverseName": "student", "DatatypeName": "stdType", "DatasetType": "INTERNAL", "GroupName": "Default.student.ugdstd", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:32 PST 2023", "DatasetId": 110, "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv19/cross-dv19.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv19/cross-dv19.1.adm
index 6c7097a..2c50ddc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv19/cross-dv19.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/cross-dataverse/cross-dv19/cross-dv19.1.adm
@@ -1,7 +1,7 @@
-{ "DataverseName": "test1", "DatasetName": "TwitterData", "DatatypeDataverseName": "test1", "DatatypeName": "Tweet", "DatasetType": "EXTERNAL", "GroupName": "Default.test1.TwitterData", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "ExternalDetails": { "DatasourceAdapter": "localfs", "Properties": [ { "Name": "path", "Value": "asterix_nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ], "LastRefreshTime": datetime("2023-11-10T07:39:34.975"), "TransactionState": 0 }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 120, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t1", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test1.t1", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:33 PST 2023", "DatasetId": 114, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t2", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test1.t2", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 117, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t3", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test1.t3", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 118, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t2", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test2.t2", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:33 PST 2023", "DatasetId": 115, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t3", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test2.t3", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 116, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t4", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test2.t4", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 119, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "test1", "DatasetName": "TwitterData", "DatatypeDataverseName": "test1", "DatatypeName": "Tweet", "DatasetType": "EXTERNAL", "GroupName": "Default.test1.TwitterData", "ExternalDetails": { "DatasourceAdapter": "localfs", "Properties": [ { "Name": "path", "Value": "asterix_nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ], "LastRefreshTime": datetime("2023-11-10T07:39:34.975"), "TransactionState": 0 }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 120, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t1", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test1.t1", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:33 PST 2023", "DatasetId": 114, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t2", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test1.t2", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 117, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t3", "DatatypeDataverseName": "test1", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test1.t3", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 118, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t2", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test2.t2", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:33 PST 2023", "DatasetId": 115, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t3", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test2.t3", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 116, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t4", "DatatypeDataverseName": "test2", "DatatypeName": "testtype", "DatasetType": "INTERNAL", "GroupName": "Default.test2.t4", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Thu Nov 09 23:39:34 PST 2023", "DatasetId": 119, "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/user-defined-functions/udf23/udf23.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/user-defined-functions/udf23/udf23.1.adm
index 784e825..6f8a556 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/user-defined-functions/udf23/udf23.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cloud/user-defined-functions/udf23/udf23.1.adm
@@ -1,6 +1,6 @@
-{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "CompactionPolicy" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "CompactionPolicy" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 13, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
-{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Database", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatabaseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ] ], "PrimaryKey": [ [ "DatabaseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 18, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
-{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Dataset", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 2, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
-{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 8, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
-{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Datatype", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatatypeName" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatatypeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 3, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
-{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Dataverse", "DatatypeDataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "CompactionPolicy": "concurrent", "CompactionPolicyProperties": [ { "Name": "max-component-count", "Value": "30" }, { "Name": "min-merge-component-count", "Value": "3" }, { "Name": "max-merge-component-count", "Value": "10" }, { "Name": "size-ratio", "Value": "1.2" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 1, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
+{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DatatypeDataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "CompactionPolicy" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "CompactionPolicy" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 13, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
+{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Database", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatabaseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ] ], "PrimaryKey": [ [ "DatabaseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 18, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
+{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Dataset", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatasetName" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatasetName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 2, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
+{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "Name" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "Name" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 8, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
+{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Datatype", "DatatypeDataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatatypeName" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ], [ "DatatypeName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 3, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
+{ "DatabaseName": "System", "DataverseName": "Metadata", "DatasetName": "Dataverse", "DatatypeDataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "GroupName": "MetadataGroup", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "DatabaseName" ], [ "DataverseName" ] ], "PrimaryKey": [ [ "DatabaseName" ], [ "DataverseName" ] ], "Autogenerated": false }, "Hints": {{ }}, "Timestamp": "Tue Oct 03 12:04:25 PDT 2023", "DatasetId": 1, "PendingOp": 0, "DatatypeDatabaseName": "System", "DatasetFormat": { "Format": "ROW" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.ast
index c3f5b77..09b313a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.ast
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv02/cross-dv02.3.ast
@@ -32,16 +32,6 @@
GroupName
FieldAccessor [
Variable [ Name=$l ]
- Field=CompactionPolicy
-]
-CompactionPolicy
-FieldAccessor [
- Variable [ Name=$l ]
- Field=CompactionPolicyProperties
-]
-CompactionPolicyProperties
-FieldAccessor [
- Variable [ Name=$l ]
Field=InternalDetails
]
InternalDetails
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.ast
index c3f5b77..09b313a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.ast
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv04/cross-dv04.3.ast
@@ -32,16 +32,6 @@
GroupName
FieldAccessor [
Variable [ Name=$l ]
- Field=CompactionPolicy
-]
-CompactionPolicy
-FieldAccessor [
- Variable [ Name=$l ]
- Field=CompactionPolicyProperties
-]
-CompactionPolicyProperties
-FieldAccessor [
- Variable [ Name=$l ]
Field=InternalDetails
]
InternalDetails
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.ast
index b3796f1..26e97e6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.ast
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv19/cross-dv19.3.ast
@@ -32,16 +32,6 @@
GroupName
FieldAccessor [
Variable [ Name=$l ]
- Field=CompactionPolicy
-]
-CompactionPolicy
-FieldAccessor [
- Variable [ Name=$l ]
- Field=CompactionPolicyProperties
-]
-CompactionPolicyProperties
-FieldAccessor [
- Variable [ Name=$l ]
Field=ExternalDetails
]
ExternalDetails
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/user-defined-functions/udf23/udf23.3.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/user-defined-functions/udf23/udf23.3.ast
index cf9ed5f..18ceb10 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/user-defined-functions/udf23/udf23.3.ast
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/user-defined-functions/udf23/udf23.3.ast
@@ -1,7 +1,76 @@
DataverseUse test
Query:
-SELECT ELEMENT [
-Variable [ Name=$l ]
+SELECT [
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatabaseName
+]
+DatabaseName
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DataverseName
+]
+DataverseName
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatasetName
+]
+DatasetName
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatatypeDataverseName
+]
+DatatypeDataverseName
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatatypeName
+]
+DatatypeName
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatasetType
+]
+DatasetType
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=GroupName
+]
+GroupName
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=InternalDetails
+]
+InternalDetails
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=Hints
+]
+Hints
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=Timestamp
+]
+Timestamp
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatasetId
+]
+DatasetId
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=PendingOp
+]
+PendingOp
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatatypeDatabaseName
+]
+DatatypeDatabaseName
+FieldAccessor [
+ Variable [ Name=$l ]
+ Field=DatasetFormat
+]
+DatasetFormat
]
FROM [ FunctionCall asterix.dataset@1[
LiteralExpr [STRING] [Metadata.Dataset]
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index a1ca571..6fe0eb8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -69,7 +69,8 @@
STORAGE_COLUMN_MAX_LEAF_NODE_SIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(10, MEGABYTE)),
STORAGE_FORMAT(STRING, "row"),
STORAGE_PARTITIONING(STRING, "dynamic"),
- STORAGE_PARTITIONS_COUNT(INTEGER, 8);
+ STORAGE_PARTITIONS_COUNT(INTEGER, 8),
+ STORAGE_MAX_COMPONENT_SIZE(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(1, StorageUtil.StorageUnit.TERABYTE));
private final IOptionType interpreter;
private final Object defaultValue;
@@ -159,6 +160,8 @@
case STORAGE_PARTITIONS_COUNT:
return "The number of storage partitions to use for static partitioning. This value should not be"
+ " changed after any dataset has been created";
+ case STORAGE_MAX_COMPONENT_SIZE:
+ return "The resultant disk component after a merge must not exceed the specified maximum size.";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -319,4 +322,8 @@
public int getStoragePartitionsCount() {
return accessor.getInt(Option.STORAGE_PARTITIONS_COUNT);
}
+
+ public long getStorageMaxComponentSize() {
+ return accessor.getLong(Option.STORAGE_MAX_COMPONENT_SIZE);
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/SizeBoundedConcurrentMergePolicy.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/SizeBoundedConcurrentMergePolicy.java
new file mode 100644
index 0000000..1384f39
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/SizeBoundedConcurrentMergePolicy.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.storage;
+
+import static org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicy.Range.isRangeMergable;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+
+/**
+ * This merge policy deals with the constraint that the resultant merged component
+ * should not exceed the specified maxResultantComponentSize.
+ */
+public class SizeBoundedConcurrentMergePolicy implements ILSMMergePolicy {
+ private final long maxComponentSize;
+ private int minMergeComponentCount;
+ private int maxMergeComponentCount;
+ private int maxComponentCount;
+ private double sizeRatio;
+
+ public SizeBoundedConcurrentMergePolicy(long maxComponentSize) {
+ this.maxComponentSize = maxComponentSize;
+ }
+
+ @Override
+ public void diskComponentAdded(ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException {
+ if (fullMergeIsRequested) {
+ throw new IllegalArgumentException("SizeBoundedConcurrentMergePolicy does not support fullMerge.");
+ }
+
+ scheduleMerge(index, false);
+ }
+
+ @Override
+ public void configure(Map<String, String> properties) {
+ minMergeComponentCount =
+ Integer.parseInt(properties.get(SizeBoundedConcurrentMergePolicyFactory.MIN_MERGE_COMPONENT_COUNT));
+ maxMergeComponentCount =
+ Integer.parseInt(properties.get(SizeBoundedConcurrentMergePolicyFactory.MAX_MERGE_COMPONENT_COUNT));
+ sizeRatio = Double.parseDouble(properties.get(SizeBoundedConcurrentMergePolicyFactory.SIZE_RATIO));
+ maxComponentCount =
+ Integer.parseInt(properties.get(SizeBoundedConcurrentMergePolicyFactory.MAX_COMPONENT_COUNT));
+ }
+
+ // hold flush when the componentCount reaches maxComponentCount, and try to find the suitable range
+ @Override
+ public boolean isMergeLagging(ILSMIndex index) throws HyracksDataException {
+ List<ILSMDiskComponent> diskComponents = index.getDiskComponents();
+ MergableSolution mergableSolutionWithComponentCount =
+ getMergableIndexesRange(index.getDiskComponents(), minMergeComponentCount, true);
+ if (mergableSolutionWithComponentCount.getComponentsCount() < maxComponentCount) {
+ // not reach the component threshold, simply return false
+ return false;
+ } else {
+ if (diskComponents.stream().anyMatch(d -> d.getState() == ILSMComponent.ComponentState.READABLE_MERGING)) {
+ // reach the component threshold and some components are being merged, return true (stop flushing)
+ return true;
+ } else {
+ // reach the component threshold but no components are being merged
+ // this can happen in two cases: (1) the system just recovers; (2) maxComponentCount is too small
+ if (!diskComponents.stream()
+ .allMatch(d -> d.getState() == ILSMComponent.ComponentState.READABLE_UNWRITABLE)) {
+ throw new IllegalStateException("Illegal disk component states in isMergeLagging");
+ }
+
+ if (scheduleMerge(index, false)) {
+ return true;
+ }
+ // desperate request to merge any two disk components, while satisfying the maxComponentSize criterion
+ // if unable to schedule a merge, don't block the component flush.
+ return scheduleMerge(index, true);
+ }
+ }
+ }
+
+ // In case of no-force merge, we try to find the largestIndexRange which satisfies the merge-policy rules.
+ // but in case of force, there is desperate request for merge, hence try finding as low as 2 components to merge.
+ private boolean scheduleMerge(ILSMIndex index, boolean force) throws HyracksDataException {
+ List<ILSMDiskComponent> diskComponents = index.getDiskComponents();
+ MergableSolution mergableSolutionWithRange;
+ if (force) {
+ mergableSolutionWithRange = getMergableIndexesRange(diskComponents, 2, false);
+ } else {
+ mergableSolutionWithRange = getMergableIndexesRange(diskComponents, minMergeComponentCount, false);
+ }
+ if (mergableSolutionWithRange.getRange() != null) {
+ triggerScheduleMerge(index, diskComponents, mergableSolutionWithRange.getRange().getStartIndex(),
+ mergableSolutionWithRange.getRange().getEndIndex());
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ // getMergableIndexesRange gives the [startComponentIndex, endComponentIndex] which can be merged based on the provided condition
+ // 1. localMinMergeComponentCount <= range(startComponentIndex, endComponentIndex) <= maxMergeComponentCount
+ // 2. sumOfComponents(startComponentIndex, endComponentIndex) < maxComponentSize
+ // In order to satisfy range(startComponentIndex, endComponentIndex) <= maxMergeComponentCount, the search range is bounded by leftBoundary, and if we reach the maxComponentSize before
+ // reaching the leftBoundary, we get our boundary where startIndex > leftBoundary.
+ // but in case the components in the range does not contribute enough to exceed maxComponentSize then the candidate range
+ // will be [leftBoundary, endComponentIndex] which satisfies both 1 & 2.
+ private MergableSolution getMergableIndexesRange(List<ILSMDiskComponent> diskComponents,
+ int localMinMergeComponentCount, boolean getMergableComponentsCount) {
+ int numComponents = diskComponents.size();
+ int candidateComponentsCount = 0;
+ for (; candidateComponentsCount < numComponents; candidateComponentsCount++) {
+ if (diskComponents.get(candidateComponentsCount)
+ .getState() != ILSMComponent.ComponentState.READABLE_UNWRITABLE) {
+ break;
+ }
+ }
+
+ if (candidateComponentsCount < localMinMergeComponentCount) {
+ return new MergableSolution(null, 0);
+ }
+
+ // count the total number of non-overlapping components that can be merged.
+ // eg: if there are two possible ranges, total non-overlapping components will be [0,2],[4,7] = 3 + 4 = 7
+ int totalMergableComponentsCount = 0;
+
+ int endComponentIndex = candidateComponentsCount - 1;
+ while (endComponentIndex >= localMinMergeComponentCount - 1) {
+ // acting as a branching condition for advancing the endComponentIndex to prevRange's startIndex - 1.
+ // thus allowing to get non-overlapping ranges.
+ // TODO(merge-policy): Need refactoring for better readability
+ boolean aRangeFound = false;
+
+ long lastComponentSize = diskComponents.get(endComponentIndex).getComponentSize();
+ long resultantComponentSize = lastComponentSize;
+
+ int probableStartIndex = endComponentIndex - 1;
+ int leftBoundaryIndex = Math.max(endComponentIndex - maxMergeComponentCount, 0);
+
+ for (; probableStartIndex >= leftBoundaryIndex; probableStartIndex--) {
+ long currentComponentSize = diskComponents.get(probableStartIndex).getComponentSize();
+ resultantComponentSize += currentComponentSize;
+
+ if (resultantComponentSize >= maxComponentSize) {
+ // since the resultComponentSize after merging index from [probableStartIndex, endComponentIndex]
+ // exceeds the maxComponentSize, the appropriate range should be (probableStartIndex, endComponentIndex]
+ if (isRangeMergable(probableStartIndex + 1, endComponentIndex, localMinMergeComponentCount,
+ (resultantComponentSize - currentComponentSize - lastComponentSize), sizeRatio,
+ lastComponentSize)) {
+ int mergableRangeCount = endComponentIndex - probableStartIndex;
+ if (!getMergableComponentsCount) {
+ return new MergableSolution(new Range(probableStartIndex + 1, endComponentIndex),
+ mergableRangeCount);
+ } else {
+ aRangeFound = true;
+ endComponentIndex = probableStartIndex;
+ totalMergableComponentsCount += mergableRangeCount;
+ }
+ }
+ // break as we already exceeded the maxComponentSize, and still haven't found the suitable range,
+ break;
+ }
+ }
+
+ // since we exceeded the leftBoundary, and still we are within maxComponentCount
+ // [leftBoundaryIndex, endComponentIndex] can be a probable solution
+ if (probableStartIndex == leftBoundaryIndex - 1) {
+ if (isRangeMergable(leftBoundaryIndex, endComponentIndex, localMinMergeComponentCount,
+ (resultantComponentSize - lastComponentSize), sizeRatio, lastComponentSize)) {
+ int mergableRangeCount = endComponentIndex - leftBoundaryIndex + 1;
+ if (!getMergableComponentsCount) {
+ return new MergableSolution(new Range(leftBoundaryIndex, endComponentIndex),
+ mergableRangeCount);
+ } else {
+ aRangeFound = true;
+ endComponentIndex = probableStartIndex;
+ totalMergableComponentsCount += mergableRangeCount;
+ }
+ }
+ }
+
+ if (!aRangeFound) {
+ --endComponentIndex;
+ }
+ }
+
+ return new MergableSolution(null, totalMergableComponentsCount);
+ }
+
+ private void triggerScheduleMerge(ILSMIndex index, List<ILSMDiskComponent> diskComponents, int startIndex,
+ int endIndex) throws HyracksDataException {
+ List<ILSMDiskComponent> mergableComponents = diskComponents.subList(startIndex, endIndex + 1);
+ index.createAccessor(NoOpIndexAccessParameters.INSTANCE).scheduleMerge(mergableComponents);
+ }
+
+ static class MergableSolution {
+ private final Range range;
+ private final int componentsCount;
+
+ MergableSolution(Range range, int componentsCount) {
+ this.range = range;
+ this.componentsCount = componentsCount;
+ }
+
+ public Range getRange() {
+ return range;
+ }
+
+ public int getComponentsCount() {
+ return componentsCount;
+ }
+ }
+
+ static class Range {
+ private final int startIndex;
+ private final int endIndex;
+
+ public Range(int startIndex, int endIndex) {
+ this.startIndex = startIndex;
+ this.endIndex = endIndex;
+ }
+
+ public int getStartIndex() {
+ return startIndex;
+ }
+
+ public int getEndIndex() {
+ return endIndex;
+ }
+
+ static boolean isRangeMergable(int startIndexInclusive, int endIndexInclusive, int localMinMergeComponentCount,
+ long resultantComponentSizeExcludingEndComponent, double sizeRatio, long endComponentSize) {
+ int mergableRangeCount = endIndexInclusive - startIndexInclusive + 1;
+ return mergableRangeCount >= localMinMergeComponentCount
+ && ((resultantComponentSizeExcludingEndComponent * sizeRatio) >= endComponentSize);
+ }
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/SizeBoundedConcurrentMergePolicyFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/SizeBoundedConcurrentMergePolicyFactory.java
new file mode 100644
index 0000000..56c0b3d
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/SizeBoundedConcurrentMergePolicyFactory.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.storage;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+public class SizeBoundedConcurrentMergePolicyFactory implements ILSMMergePolicyFactory {
+
+ private static final long serialVersionUID = 1L;
+ public static final String NAME = "size-bounded-concurrent";
+ public static final String MIN_MERGE_COMPONENT_COUNT = "min-merge-component-count";
+ public static final String MAX_MERGE_COMPONENT_COUNT = "max-merge-component-count";
+ public static final String MAX_COMPONENT_COUNT = "max-component-count";
+ public static final String SIZE_RATIO = "size-ratio";
+ public static final Set<String> PROPERTIES_NAMES =
+ Set.of(MIN_MERGE_COMPONENT_COUNT, MAX_MERGE_COMPONENT_COUNT, SIZE_RATIO, MAX_COMPONENT_COUNT);
+
+ @Override
+ public ILSMMergePolicy createMergePolicy(Map<String, String> configuration, INCServiceContext ctx) {
+ long maxComponentSize = ((INcApplicationContext) ctx.getApplicationContext()).getStorageProperties()
+ .getStorageMaxComponentSize();
+ ILSMMergePolicy policy = new SizeBoundedConcurrentMergePolicy(maxComponentSize);
+ policy.configure(configuration);
+ return policy;
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Set<String> getPropertiesNames() {
+ return PROPERTIES_NAMES;
+ }
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ return registry.getClassIdentifier(getClass(), serialVersionUID);
+ }
+
+ @SuppressWarnings("squid:S1172") // unused parameter
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
+ return new SizeBoundedConcurrentMergePolicyFactory();
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index eb39f23..bd67b11 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -21,9 +21,9 @@
import java.util.LinkedHashMap;
import java.util.Map;
+import org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicyFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
-import org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFactory;
/**
* A static class that stores storage constants
@@ -46,7 +46,7 @@
public static final String MASK_FILE_PREFIX = ".mask_";
public static final String COMPONENT_MASK_FILE_PREFIX = MASK_FILE_PREFIX + "C_";
public static final float DEFAULT_TREE_FILL_FACTOR = 1.00f;
- public static final String DEFAULT_COMPACTION_POLICY_NAME = ConcurrentMergePolicyFactory.NAME;
+ public static final String DEFAULT_COMPACTION_POLICY_NAME = SizeBoundedConcurrentMergePolicyFactory.NAME;
public static final String DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME = "correlated-prefix";
public static final Map<String, String> DEFAULT_COMPACTION_POLICY_PROPERTIES;
public static final int METADATA_PARTITION = -1;
@@ -64,10 +64,12 @@
static {
DEFAULT_COMPACTION_POLICY_PROPERTIES = new LinkedHashMap<>();
- DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.MAX_COMPONENT_COUNT, "30");
- DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.MIN_MERGE_COMPONENT_COUNT, "3");
- DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.MAX_MERGE_COMPONENT_COUNT, "10");
- DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.SIZE_RATIO, "1.2");
+ DEFAULT_COMPACTION_POLICY_PROPERTIES.put(SizeBoundedConcurrentMergePolicyFactory.MIN_MERGE_COMPONENT_COUNT,
+ "3");
+ DEFAULT_COMPACTION_POLICY_PROPERTIES.put(SizeBoundedConcurrentMergePolicyFactory.MAX_MERGE_COMPONENT_COUNT,
+ "10");
+ DEFAULT_COMPACTION_POLICY_PROPERTIES.put(SizeBoundedConcurrentMergePolicyFactory.SIZE_RATIO, "1.2");
+ DEFAULT_COMPACTION_POLICY_PROPERTIES.put(SizeBoundedConcurrentMergePolicyFactory.MAX_COMPONENT_COUNT, "30");
}
private StorageConstants() {
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/SizeBoundedConcurrentMergePolicyTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/SizeBoundedConcurrentMergePolicyTest.java
new file mode 100644
index 0000000..b6f8450
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/SizeBoundedConcurrentMergePolicyTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicy;
+import org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicyFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class SizeBoundedConcurrentMergePolicyTest {
+
+ private static final int MIN_MERGE_COMPONENT_COUNT = 3;
+ private static final int MAX_MERGE_COMPONENT_COUNT = 5;
+ private static final int MAX_COMPONENT_COUNT = 10;
+ private static final long MAX_STORAGE_COMPONENT_SIZE = 30L;
+ private static final double SIZE_RATIO = 1.0;
+
+ @Test
+ public void testBasic() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(List.of(1L, 2L, 3L, 6L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ policy.diskComponentAdded(index, false);
+
+ Assert.assertEquals(sizes, resultSizes);
+ }
+
+ @Test
+ public void testNotEnoughComponents() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(1L, 2L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ policy.diskComponentAdded(index, false);
+
+ Assert.assertTrue(resultSizes.isEmpty());
+ }
+
+ @Test
+ public void testNoFullMergeAllowed() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(1L, 2L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+
+ IllegalArgumentException fullMergeException = Assert.assertThrows(IllegalArgumentException.class, () -> {
+ policy.diskComponentAdded(index, true);
+ });
+
+ Assert.assertEquals("SizeBoundedConcurrentMergePolicy does not support fullMerge.",
+ fullMergeException.getMessage());
+ }
+
+ // [ 1, 2, 3] [7] -> this is not a valid range, as (1 + 2 + 3) * 1(size-ratio) >= 7
+ // [ 1, 2] [3] -> A valid range, as 1+2+3 < 30 (max component count) && (1+2) * 1 >= 3
+ @Test
+ public void testSkipLargeComponent() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(1L, 2L, 3L, 7L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ policy.diskComponentAdded(index, false);
+
+ Assert.assertEquals(sizes.subList(0, 3), resultSizes);
+ }
+
+ @Test
+ public void testSkipMergeComponent0_2() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(1L, 2L, 17L, 20L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ policy.diskComponentAdded(index, false);
+
+ Assert.assertTrue(resultSizes.isEmpty());
+ }
+
+ @Test
+ public void testAvoidExceedingMaxSize() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(3L, 9L, 11L, 20L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ policy.diskComponentAdded(index, false);
+
+ Assert.assertEquals(sizes.subList(0, 3), resultSizes);
+ }
+
+ @Test
+ public void testNoMergableComponents() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(15L, 16L, 15L, 16L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ policy.diskComponentAdded(index, false);
+
+ Assert.assertTrue(resultSizes.isEmpty());
+ }
+
+ @Test
+ public void testConcurrentMerge() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 100L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ // component 5L is merging
+ Mockito.when(index.getDiskComponents().get(4).getState())
+ .thenReturn(ILSMComponent.ComponentState.READABLE_MERGING);
+
+ policy.diskComponentAdded(index, false);
+ Assert.assertEquals(sizes.subList(0, 4), resultSizes);
+ }
+
+ @Test
+ public void testMergeLagging() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(
+ Arrays.asList(1L, 2L, 3L, 4L, 29L, 1L, 2L, 3L, 4L, 29L, 1L, 2L, 3L, 4L, 29L, 1L, 2L, 3L, 4L, 29L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ Assert.assertTrue(policy.isMergeLagging(index));
+ Assert.assertEquals(sizes.subList(15, 19), resultSizes);
+ }
+
+ @Test
+ public void testNoMergeLagging() throws HyracksDataException {
+ List<Long> sizes = new ArrayList<>(Arrays.asList(1L, 2L, 3L, 4L, 29L, 1L, 2L, 3L, 4L));
+ List<Long> resultSizes = new ArrayList<>();
+ ILSMIndex index = mockIndex(sizes, resultSizes);
+ ILSMMergePolicy policy = createMergePolicy();
+ Assert.assertFalse(policy.isMergeLagging(index));
+ }
+
+ private ILSMMergePolicy createMergePolicy() {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(SizeBoundedConcurrentMergePolicyFactory.MIN_MERGE_COMPONENT_COUNT,
+ String.valueOf(MIN_MERGE_COMPONENT_COUNT));
+ properties.put(SizeBoundedConcurrentMergePolicyFactory.MAX_MERGE_COMPONENT_COUNT,
+ String.valueOf(MAX_MERGE_COMPONENT_COUNT));
+ properties.put(SizeBoundedConcurrentMergePolicyFactory.SIZE_RATIO, String.valueOf(SIZE_RATIO));
+ properties.put(SizeBoundedConcurrentMergePolicyFactory.MAX_COMPONENT_COUNT,
+ String.valueOf(MAX_COMPONENT_COUNT));
+
+ ILSMMergePolicy policy = new SizeBoundedConcurrentMergePolicy(MAX_STORAGE_COMPONENT_SIZE);
+ policy.configure(properties);
+ return policy;
+ }
+
+ private ILSMIndex mockIndex(List<Long> componentSizes, List<Long> mergedSizes) throws HyracksDataException {
+ List<ILSMDiskComponent> components = new ArrayList<>();
+ for (Long size : componentSizes) {
+ ILSMDiskComponent component = Mockito.mock(ILSMDiskComponent.class);
+ Mockito.when(component.getComponentSize()).thenReturn(size);
+ Mockito.when(component.getState()).thenReturn(ILSMComponent.ComponentState.READABLE_UNWRITABLE);
+ components.add(component);
+ }
+
+ ILSMIndex index = Mockito.mock(ILSMIndex.class);
+ Mockito.when(index.getDiskComponents()).thenReturn(components);
+
+ ILSMIndexAccessor accessor = Mockito.mock(ILSMIndexAccessor.class);
+
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ List<ILSMDiskComponent> mergedComponents = invocation.getArgument(0);
+ mergedComponents.forEach(component -> {
+ mergedSizes.add(component.getComponentSize());
+ });
+ return null;
+ }
+ }).when(accessor).scheduleMerge(Mockito.anyListOf(ILSMDiskComponent.class));
+
+ Mockito.when(index.createAccessor(Mockito.any(IIndexAccessParameters.class))).thenReturn(accessor);
+
+ return index;
+ }
+}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
index 7bb6f11..95b53a8 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
@@ -122,7 +122,7 @@
AUnionType.createUnknownableType(BuiltinType.ADOUBLE),
AUnionType.createUnknownableType(BuiltinType.AINT64),
AUnionType.createUnknownableType(BuiltinType.AINT64),
- AUnionType.createUnknownableType(BuiltinType.AINT64) };
+ AUnionType.createUnknownableType(BuiltinType.AINT64), };
final ARecordType parameters =
new ARecordType(MERGE_POLICY_PARAMETERS_PARAMETER_NAME, parameterNames, parametersTypes, false);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 843d7c3..71458f9 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -24,6 +24,7 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.asterix.common.api.ILSMComponentIdGeneratorFactory;
import org.apache.asterix.common.api.INamespacePathResolver;
@@ -43,6 +44,7 @@
import org.apache.asterix.common.ioopcallbacks.LSMIndexPageWriteCallbackFactory;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.metadata.MetadataUtil;
+import org.apache.asterix.common.storage.SizeBoundedConcurrentMergePolicyFactory;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.adapter.factory.GenericAdapterFactory;
@@ -285,7 +287,8 @@
throws AlgebricksException {
String[] builtInCompactionPolicyClassNames = new String[] { ConstantMergePolicyFactory.class.getName(),
PrefixMergePolicyFactory.class.getName(), ConcurrentMergePolicyFactory.class.getName(),
- NoMergePolicyFactory.class.getName(), CorrelatedPrefixMergePolicyFactory.class.getName() };
+ NoMergePolicyFactory.class.getName(), CorrelatedPrefixMergePolicyFactory.class.getName(),
+ SizeBoundedConcurrentMergePolicyFactory.class.getName() };
for (String policyClassName : builtInCompactionPolicyClassNames) {
CompactionPolicy compactionPolicy = getCompactionPolicyEntity(policyClassName);
MetadataManager.INSTANCE.addCompactionPolicy(mdTxnCtx, compactionPolicy);
@@ -305,6 +308,13 @@
CompactionPolicy compactionPolicy = getCompactionPolicyEntity(ConcurrentMergePolicyFactory.class.getName());
MetadataManager.INSTANCE.addCompactionPolicy(mdTxnCtx, compactionPolicy);
}
+
+ if (MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx, MetadataConstants.SYSTEM_DATABASE,
+ MetadataConstants.METADATA_DATAVERSE_NAME, SizeBoundedConcurrentMergePolicyFactory.NAME) == null) {
+ CompactionPolicy compactionPolicy =
+ getCompactionPolicyEntity(SizeBoundedConcurrentMergePolicyFactory.class.getName());
+ MetadataManager.INSTANCE.addCompactionPolicy(mdTxnCtx, compactionPolicy);
+ }
}
private static void insertSynonymEntitiesIfNotExist(MetadataTransactionContext mdTxnCtx,
@@ -436,15 +446,18 @@
if (createMetadataDataset) {
final double bloomFilterFalsePositiveRate =
appContext.getStorageProperties().getBloomFilterFalsePositiveRate();
+ Map<String, String> defaultCompactionPolicyProperties =
+ StorageConstants.DEFAULT_COMPACTION_POLICY_PROPERTIES;
+
LSMBTreeLocalResourceFactory lsmBtreeFactory =
new LSMBTreeLocalResourceFactory(storageComponentProvider.getStorageManager(), typeTraits,
cmpFactories, null, null, null, opTrackerFactory, ioOpCallbackFactory,
pageWriteCallbackFactory, storageComponentProvider.getMetadataPageManagerFactory(),
new AsterixVirtualBufferCacheProvider(datasetId),
storageComponentProvider.getIoOperationSchedulerProvider(),
- appContext.getMetadataMergePolicyFactory(),
- StorageConstants.DEFAULT_COMPACTION_POLICY_PROPERTIES, true, bloomFilterKeyFields,
- bloomFilterFalsePositiveRate, true, null, NoOpCompressorDecompressorFactory.INSTANCE, true,
+ appContext.getMetadataMergePolicyFactory(), defaultCompactionPolicyProperties, true,
+ bloomFilterKeyFields, bloomFilterFalsePositiveRate, true, null,
+ NoOpCompressorDecompressorFactory.INSTANCE, true,
TypeTraitProvider.INSTANCE.getTypeTrait(BuiltinType.ANULL), NullIntrospector.INSTANCE,
false, appContext.isCloudDeployment());
DatasetLocalResourceFactory dsLocalResourceFactory =