[ASTERIXDB-3428][OTH] Improve storage logging
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Storage log events will be in storage_debug.log
- Reduce schema logging on query
- Several improvements when logging on merge/flush
Change-Id: I8b4e92585dee55a7f6a24342fb8d1a6390449c1c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18368
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
index 329ccaf..466b617 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
@@ -47,7 +47,7 @@
}
private boolean haltOnFailure(ILSMIOOperation operation) {
- switch (operation.getIOOpertionType()) {
+ switch (operation.getIOOperationType()) {
case CLEANUP:
case REPLICATE:
return false;
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index 5d734f8..e0eb88d 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -45,7 +45,7 @@
[cc]
address = 127.0.0.1
app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
-heartbeat.period=2000
+heartbeat.period=20000
heartbeat.max.misses=25
[common]
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
index c727f52..0d8677f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
@@ -159,20 +159,20 @@
synchronized (TestLsmIoOpCallbackFactory.this) {
if (op.getNewComponent() != null) {
if (op.getNewComponent() == EmptyComponent.INSTANCE) {
- if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (op.getIOOperationType() == LSMIOOperationType.FLUSH) {
rollbackFlushes++;
} else {
rollbackMerges++;
}
} else {
- if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (op.getIOOperationType() == LSMIOOperationType.FLUSH) {
completedFlushes++;
} else {
completedMerges++;
}
}
} else {
- recordFailure(op.getIOOpertionType());
+ recordFailure(op.getIOOperationType());
}
TestLsmIoOpCallbackFactory.this.notifyAll();
}
diff --git a/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml b/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml
index 8f81828..4e9bd7e 100644
--- a/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml
+++ b/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml
@@ -27,6 +27,9 @@
<File name="DebugLog" fileName="target/debug.log">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</File>
+ <File name="StorageDebug" fileName="target/storage_debug.log">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+ </File>
<Console name="ConsoleTest" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %c{1} - %msg%n"/>
</Console>
@@ -45,5 +48,24 @@
<Logger name="org.apache.asterix.test" level="INFO">
<AppenderRef ref="ConsoleTest"/>
</Logger>
+ <!-- Storage debug -->
+ <Logger name="org.apache.hyracks.storage" level="debug" additivity="false">
+ <AppenderRef ref="InfoLog" level="warn"/>
+ <AppenderRef ref="StorageDebug"/>
+ </Logger>
+ <Logger name="org.apache.hyracks.cloud" level="debug" additivity="false">
+ <AppenderRef ref="StorageDebug"/>
+ </Logger>
+ <Logger name="org.apache.asterix.column" level="debug" additivity="false">
+ <AppenderRef ref="StorageDebug"/>
+ </Logger>
+ <Logger name="org.apache.asterix.cloud" additivity="false">
+ <AppenderRef ref="InfoLog" level="warn"/>
+ <AppenderRef ref="StorageDebug" level="debug"/>
+ </Logger>
+ <Logger name="org.apache.asterix.common.context.GlobalVirtualBufferCache" additivity="false">
+ <AppenderRef ref="InfoLog" level="warn"/>
+ <AppenderRef ref="StorageDebug" level="debug"/>
+ </Logger>
</Loggers>
</Configuration>
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
index 72f3446..ad2392b 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
@@ -120,7 +120,7 @@
// Everything is cached, no need to invoke cloud-based accessor for read operations
accessor = new LocalAccessor(cloudClient, bucket, localIoManager);
} else {
- LOGGER.debug("The number of uncached files: {}. Uncached files: {}", remainingUncachedFiles, cloudFiles);
+ LOGGER.info("The number of uncached files: {}. Uncached files: {}", remainingUncachedFiles, cloudFiles);
// Get list of FileReferences from the list of cloud (i.e., resolve each path's string to FileReference)
List<FileReference> uncachedFiles = resolve(cloudFiles);
// Create a parallel downloader using the given cloudClient
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/ParallelCacher.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/ParallelCacher.java
index 00db458..a087ef1 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/ParallelCacher.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/ParallelCacher.java
@@ -158,11 +158,11 @@
@Override
public boolean remove(Collection<FileReference> deletedFiles) {
if (!deletedFiles.isEmpty()) {
- LOGGER.info("Deleting {}", deletedFiles);
+ LOGGER.debug("Deleting {}", deletedFiles);
}
for (FileReference fileReference : deletedFiles) {
- remove(fileReference);
+ doRemove(fileReference);
}
return isEmpty();
@@ -170,13 +170,8 @@
@Override
public boolean remove(FileReference fileReference) {
- LOGGER.info("Deleting {}", fileReference);
- if (isDataFile(fileReference)) {
- uncachedDataFiles.remove(fileReference);
- } else {
- uncachedMetadataFiles.remove(fileReference);
- }
-
+ LOGGER.debug("Deleting {}", fileReference);
+ doRemove(fileReference);
return isEmpty();
}
@@ -216,12 +211,20 @@
return DATA_FILTER.accept(null, fileReference.getName());
}
+ private void doRemove(FileReference fileReference) {
+ if (isDataFile(fileReference)) {
+ uncachedDataFiles.remove(fileReference);
+ } else {
+ uncachedMetadataFiles.remove(fileReference);
+ }
+ }
+
private synchronized boolean isEmpty() {
if (!checkEmpty) {
return false;
}
int totalSize = uncachedDataFiles.size() + uncachedMetadataFiles.size();
- LOGGER.info("Current number of uncached files {}", totalSize);
+ LOGGER.debug("Current number of uncached files {}", totalSize);
return totalSize == 0;
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index 89ea4b1..67ce2c8 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -49,7 +49,7 @@
// First get the set of local files
Set<FileReference> localFiles = ioManager.list(partitionPath);
Iterator<FileReference> localFilesIter = localFiles.iterator();
- LOGGER.info("Cleaning partition {}.", partitionPath.getRelativePath());
+ LOGGER.debug("Cleaning partition {}.", partitionPath.getRelativePath());
// Reconcile local files and cloud files
while (localFilesIter.hasNext()) {
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
index a966d61..2686204 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
@@ -19,6 +19,8 @@
package org.apache.asterix.column.operation.lsm.flush;
import static org.apache.asterix.column.util.ColumnValuesUtil.getNormalizedTypeTag;
+import static org.apache.asterix.column.util.SchemaConstants.META_RECORD_SCHEMA;
+import static org.apache.asterix.column.util.SchemaConstants.RECORD_SCHEMA;
import java.io.ByteArrayInputStream;
import java.io.DataInput;
@@ -582,11 +584,10 @@
// This should be a low frequency object creation
SchemaStringBuilderVisitor schemaBuilder = new SchemaStringBuilderVisitor(fieldNamesDictionary);
String recordSchema = LogRedactionUtil.userData(schemaBuilder.build(root));
- LOGGER.debug("Schema for {} has changed: \n {}", SchemaStringBuilderVisitor.RECORD_SCHEMA, recordSchema);
+ LOGGER.debug("Schema for {} has changed: \n {}", RECORD_SCHEMA, recordSchema);
if (metaRoot != null) {
String metaRecordSchema = LogRedactionUtil.userData(schemaBuilder.build(metaRoot));
- LOGGER.debug("Schema for {} has changed: \n {}", SchemaStringBuilderVisitor.META_RECORD_SCHEMA,
- metaRecordSchema);
+ LOGGER.debug("Schema for {} has changed: \n {}", META_RECORD_SCHEMA, metaRecordSchema);
}
}
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
index 67631cd..5028706 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.column.operation.query;
+import static org.apache.asterix.column.util.SchemaConstants.LOG_PROBABILITY;
+import static org.apache.asterix.column.util.SchemaConstants.RECORD_SCHEMA;
import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.ALL_FIELDS_TYPE;
import java.io.ByteArrayInputStream;
@@ -224,7 +226,7 @@
// log normalized filter
logFilter(jobId, normalizedFilterEvaluator, normalizedEvaluatorFactory.toString());
// log requested schema
- logSchema(jobId, clippedRoot, SchemaStringBuilderVisitor.RECORD_SCHEMA, fieldNamesDictionary);
+ logSchema(jobId, clippedRoot, RECORD_SCHEMA, fieldNamesDictionary);
// Primary key readers
PrimitiveColumnValuesReader[] primaryKeyReaders =
@@ -270,7 +272,7 @@
protected static void logSchema(String jobId, ObjectSchemaNode root, String schemaSource,
IFieldNamesDictionary fieldNamesDictionary) throws HyracksDataException {
- if (jobId != null && LOGGER.isDebugEnabled()) {
+ if (jobId != null && LOGGER.isDebugEnabled() && System.nanoTime() % LOG_PROBABILITY == 0) {
SchemaStringBuilderVisitor schemaBuilder = new SchemaStringBuilderVisitor(fieldNamesDictionary);
String schema = LogRedactionUtil.userData(schemaBuilder.build(root));
LOGGER.debug("Queried {} schema [{}]: \n {}", schemaSource, jobId, schema);
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
index 1869415..356ddaa 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.column.operation.query;
+import static org.apache.asterix.column.util.SchemaConstants.META_RECORD_SCHEMA;
+import static org.apache.asterix.column.util.SchemaConstants.RECORD_SCHEMA;
+
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
@@ -40,7 +43,6 @@
import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
-import org.apache.asterix.column.util.SchemaStringBuilderVisitor;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.IColumnValuesReaderFactory;
import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
@@ -172,9 +174,9 @@
// log normalized filter
logFilter(jobId, normalizedFilterEvaluator, normalizedEvaluatorFactory.toString());
// log requested schema for record
- logSchema(jobId, clippedRoot, SchemaStringBuilderVisitor.RECORD_SCHEMA, fieldNamesDictionary);
+ logSchema(jobId, clippedRoot, RECORD_SCHEMA, fieldNamesDictionary);
// log requested schema for meta-record
- logSchema(jobId, metaClippedRoot, SchemaStringBuilderVisitor.META_RECORD_SCHEMA, fieldNamesDictionary);
+ logSchema(jobId, metaClippedRoot, META_RECORD_SCHEMA, fieldNamesDictionary);
// Primary key readers
PrimitiveColumnValuesReader[] primaryKeyReaders =
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaConstants.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaConstants.java
new file mode 100644
index 0000000..f3c3ee7
--- /dev/null
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaConstants.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.util;
+
+public class SchemaConstants {
+ private SchemaConstants() {
+ }
+
+ // 1/50th
+ public static final int LOG_PROBABILITY = 50;
+ public static String RECORD_SCHEMA = "record";
+ public static String META_RECORD_SCHEMA = "meta-record";
+}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaStringBuilderVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaStringBuilderVisitor.java
index 571ab83..6d991cf 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaStringBuilderVisitor.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaStringBuilderVisitor.java
@@ -40,8 +40,6 @@
import it.unimi.dsi.fastutil.ints.IntList;
public class SchemaStringBuilderVisitor implements ISchemaNodeVisitor<Void, Void> {
- public static String RECORD_SCHEMA = "record";
- public static String META_RECORD_SCHEMA = "meta-record";
private final StringBuilder builder;
private final List<String> fieldNames;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AtomicLSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AtomicLSMIOOperationCallback.java
index 111c6e7..08273a7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AtomicLSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AtomicLSMIOOperationCallback.java
@@ -42,10 +42,10 @@
if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
return;
}
- if (operation.getIOOpertionType() != LSMIOOperationType.LOAD
+ if (operation.getIOOperationType() != LSMIOOperationType.LOAD
&& operation.getAccessor().getOpContext().getOperation() == IndexOperation.DELETE_COMPONENTS) {
deleteComponentsFromCheckpoint(operation);
- } else if (operation.getIOOpertionType() == LSMIOOperationType.LOAD) {
+ } else if (operation.getIOOperationType() == LSMIOOperationType.LOAD) {
addComponentToCheckpoint(operation);
} else if (isMerge(operation)) {
IoUtil.delete(getOperationMaskFilePath(operation));
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
index e3fe663..fc895bc 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
@@ -104,15 +104,15 @@
if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
return;
}
- if (operation.getIOOpertionType() == LSMIOOperationType.LOAD) {
+ if (operation.getIOOperationType() == LSMIOOperationType.LOAD) {
Map<String, Object> map = operation.getParameters();
putComponentIdIntoMetadata(operation.getNewComponent(), (LSMComponentId) map.get(KEY_FLUSHED_COMPONENT_ID));
- } else if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ } else if (operation.getIOOperationType() == LSMIOOperationType.FLUSH) {
Map<String, Object> map = operation.getParameters();
putLSNIntoMetadata(operation.getNewComponent(), (Long) map.get(KEY_FLUSH_LOG_LSN));
putComponentIdIntoMetadata(operation.getNewComponent(),
((FlushOperation) operation).getFlushingComponent().getId());
- } else if (operation.getIOOpertionType() == LSMIOOperationType.MERGE) {
+ } else if (operation.getIOOperationType() == LSMIOOperationType.MERGE) {
List<ILSMDiskComponent> mergedComponents = operation.getAccessor().getOpContext().getComponentsToBeMerged();
putLSNIntoMetadata(operation.getNewComponent(), mergedComponents);
putComponentIdIntoMetadata(operation.getNewComponent(), mergedComponents);
@@ -128,11 +128,11 @@
if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
return;
}
- if (operation.getIOOpertionType() != LSMIOOperationType.LOAD
+ if (operation.getIOOperationType() != LSMIOOperationType.LOAD
&& operation.getAccessor().getOpContext().getOperation() == IndexOperation.DELETE_COMPONENTS) {
deleteComponentsFromCheckpoint(operation);
- } else if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH
- || operation.getIOOpertionType() == LSMIOOperationType.LOAD) {
+ } else if (operation.getIOOperationType() == LSMIOOperationType.FLUSH
+ || operation.getIOOperationType() == LSMIOOperationType.LOAD) {
addComponentToCheckpoint(operation);
} else if (isMerge(operation)) {
IoUtil.delete(getOperationMaskFilePath(operation));
@@ -144,7 +144,7 @@
FileReference target = operation.getTarget();
Map<String, Object> map = operation.getParameters();
final Long lsn =
- operation.getIOOpertionType() == LSMIOOperationType.FLUSH ? (Long) map.get(KEY_FLUSH_LOG_LSN) : 0L;
+ operation.getIOOperationType() == LSMIOOperationType.FLUSH ? (Long) map.get(KEY_FLUSH_LOG_LSN) : 0L;
final LSMComponentId id = (LSMComponentId) map.get(KEY_FLUSHED_COMPONENT_ID);
final ResourceReference ref = ResourceReference.of(target.getAbsolutePath());
final long componentSequence = IndexComponentFileReference.of(ref.getName()).getSequenceEnd();
@@ -153,7 +153,7 @@
protected void deleteComponentsFromCheckpoint(ILSMIOOperation operation) throws HyracksDataException {
// component was deleted... if a flush, do nothing.. if a merge, must update the checkpoint file
- if (operation.getIOOpertionType() == LSMIOOperationType.MERGE) {
+ if (operation.getIOOperationType() == LSMIOOperationType.MERGE) {
// Get component id of the last disk component
LSMComponentId mostRecentComponentId =
getMostRecentComponentId(operation.getAccessor().getOpContext().getComponentsToBeMerged());
@@ -161,8 +161,8 @@
FileReference target = operation.getTarget();
final ResourceReference ref = ResourceReference.of(target.getAbsolutePath());
indexCheckpointManagerProvider.get(ref).setLastComponentId(mostRecentComponentId.getMaxId());
- } else if (operation.getIOOpertionType() != LSMIOOperationType.FLUSH) {
- throw new IllegalStateException("Unexpected IO operation: " + operation.getIOOpertionType());
+ } else if (operation.getIOOperationType() != LSMIOOperationType.FLUSH) {
+ throw new IllegalStateException("Unexpected IO operation: " + operation.getIOOperationType());
}
}
@@ -258,8 +258,8 @@
@Override
public synchronized void scheduled(ILSMIOOperation operation) throws HyracksDataException {
- dsInfo.declareActiveIOOperation(operation.getIOOpertionType(), partition);
- if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ dsInfo.declareActiveIOOperation(operation.getIOOperationType(), partition);
+ if (operation.getIOOperationType() == LSMIOOperationType.FLUSH) {
pendingFlushes++;
FlushOperation flush = (FlushOperation) operation;
Map<String, Object> map = operation.getAccessor().getOpContext().getParameters();
@@ -273,7 +273,7 @@
@Override
public synchronized void completed(ILSMIOOperation operation) {
- if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (operation.getIOOperationType() == LSMIOOperationType.FLUSH) {
pendingFlushes--;
if (operation.getStatus() == LSMIOOperationStatus.SUCCESS) {
Map<String, Object> map = operation.getAccessor().getOpContext().getParameters();
@@ -281,7 +281,7 @@
pendingFlushes == 0 ? firstLsnForCurrentMemoryComponent : (Long) map.get(KEY_FLUSH_LOG_LSN);
}
}
- dsInfo.undeclareActiveIOOperation(operation.getIOOpertionType(), partition);
+ dsInfo.undeclareActiveIOOperation(operation.getIOOperationType(), partition);
}
public synchronized boolean hasPendingFlush() {
@@ -300,7 +300,7 @@
}
protected boolean isMerge(ILSMIOOperation operation) {
- return operation.getIOOpertionType() == LSMIOOperationType.MERGE
+ return operation.getIOOperationType() == LSMIOOperationType.MERGE
&& operation.getAccessor().getOpContext().getOperation() != IndexOperation.DELETE_COMPONENTS;
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationOperation.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationOperation.java
index 258f24a..ce6d6ba 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationOperation.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationOperation.java
@@ -56,7 +56,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.REPLICATE;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
index 0253b65..7010e88 100644
--- a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
+++ b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
@@ -62,6 +62,10 @@
LOGGER.info("Used space: {}, pressureCapacity: {} (isPressured: {})",
StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
true);
+ } else if (usedSpace >= diskSpace.getAllocatedCapacity()) {
+ LOGGER.warn("Allocated disk space has been exceeded. Used space: {}, pressureCapacity: {}",
+ StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
+ true);
} else {
LOGGER.debug("Used space: {}, pressureCapacity: {} (isPressured: {})",
StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 3a0f2b9..5e241c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -237,6 +237,6 @@
@Override
public String toString() {
- return "{\"class\":\"" + getClass().getSimpleName() + "\",\"file\":\"" + file.getRelativePath() + "\"}";
+ return "{\"class\": \"" + getClass().getSimpleName() + "\", \"file\": \"" + file.getRelativePath() + "\"}";
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
index 8d3ae32..6dde7e9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
@@ -72,7 +72,7 @@
private IChainedComponentBulkLoader createColumnIndexBulkLoader(ILSMIOOperation operation, float fillFactor,
boolean verifyInput, IPageWriteCallback callback) throws HyracksDataException {
- LSMIOOperationType operationType = operation.getIOOpertionType();
+ LSMIOOperationType operationType = operation.getIOOperationType();
LSMColumnBTree lsmColumnBTree = (LSMColumnBTree) getLsmIndex();
ColumnBTree columnBTree = (ColumnBTree) getIndex();
IColumnMetadata columnMetadata;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
index b3da5f2..958923f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
@@ -91,5 +91,9 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index e2e49e3..705701e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -38,9 +38,11 @@
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
-
+ private static final Logger LOGGER = LogManager.getLogger();
private static final FilenameFilter BTREE_FILTER =
(dir, name) -> !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
@@ -97,6 +99,8 @@
validateFiles(btreeFilesSet, allBloomFilterFiles, BLOOM_FILTER_FILTER, null, btreeFactory.getBufferCache());
// Sanity check.
if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
+ LOGGER.error("Unequal number of trees and filters. Trees: {}, Filters: {}", allBTreeFiles,
+ allBloomFilterFiles);
throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
index 822df6f..278b90f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
@@ -74,7 +74,7 @@
/**
* @return the operation type
*/
- LSMIOOperationType getIOOpertionType();
+ LSMIOOperationType getIOOperationType();
@Override
LSMIOOperationStatus call() throws HyracksDataException;
@@ -92,8 +92,7 @@
/**
* clean up left over files in case of an exception during execution
*
- * @param bufferCache
- * a buffercache that manages the files
+ * @param bufferCache a buffer cache that manages the files
*/
void cleanup(IBufferCache bufferCache);
@@ -157,7 +156,6 @@
Map<String, Object> getParameters();
/**
- *
* @return the estimated number of disk pages remaining for this IO operation
*/
long getRemainingPages();
@@ -173,7 +171,6 @@
void pause();
/**
- *
* @return whether this IO operation is currently active (i.e., not paused)
*/
boolean isActive();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java
index 3a8e558..602ea14 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java
@@ -52,7 +52,7 @@
@Override
public void scheduleOperation(ILSMIOOperation operation) {
- switch (operation.getIOOpertionType()) {
+ switch (operation.getIOOperationType()) {
case FLUSH:
scheduleFlush(operation);
break;
@@ -70,13 +70,13 @@
default:
// this should never happen
// just guard here to avoid silent failures in case of future extensions
- throw new IllegalArgumentException("Unknown operation type " + operation.getIOOpertionType());
+ throw new IllegalArgumentException("Unknown operation type " + operation.getIOOperationType());
}
}
@Override
public void completeOperation(ILSMIOOperation operation) throws HyracksDataException {
- switch (operation.getIOOpertionType()) {
+ switch (operation.getIOOperationType()) {
case FLUSH:
completeFlush(operation);
break;
@@ -91,7 +91,7 @@
default:
// this should never happen
// just guard here to avoid silent failures in case of future extensions
- throw new IllegalArgumentException("Unknown operation type " + operation.getIOOpertionType());
+ throw new IllegalArgumentException("Unknown operation type " + operation.getIOOperationType());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
index a5f70d9..8ae0571 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
@@ -219,4 +219,10 @@
}
}
}
+
+ @Override
+ public String toString() {
+ String fileName = target != null ? "\"fileName\": \"" + target.getFile().getName() + "\", " : "";
+ return "{" + fileName + "\"ioOpID\": " + System.identityHashCode(this) + "}";
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
index 966d6d1..baf0931 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
@@ -153,8 +153,7 @@
/**
* Mark the component as valid
*
- * @param persist
- * whether the call should force data to disk before returning
+ * @param persist whether the call should force data to disk before returning
* @throws HyracksDataException
*/
@Override
@@ -234,7 +233,7 @@
if (withFilter && getLsmIndex().getFilterFields() != null) {
chainedBulkLoader.addBulkLoader(createFilterBulkLoader());
}
- IChainedComponentBulkLoader indexBulkloader = operation.getIOOpertionType() == LSMIOOperationType.MERGE
+ IChainedComponentBulkLoader indexBulkloader = operation.getIOOperationType() == LSMIOOperationType.MERGE
? createMergeIndexBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, callback)
: createIndexBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, callback);
chainedBulkLoader.addBulkLoader(indexBulkloader);
@@ -245,7 +244,6 @@
@Override
public String toString() {
- return "{\"class\":" + getClass().getSimpleName() + "\", \"id\":" + componentId + ", \"index\":" + getIndex()
- + "}";
+ return "{\"id\": " + componentId + ", index:" + getIndex() + "}";
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index f7a15fb..25d928c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -728,10 +728,9 @@
@Override
public final String toString() {
- return "{\"class\" : \"" + getClass().getSimpleName() + "\", \"dir\" : \"" + fileManager.getBaseDir()
- + "\", \"memory\" : " + (memoryComponents == null ? 0 : memoryComponents) + ", \"disk\" : "
- + diskComponents.size() + ", \"num-scheduled-flushes\":" + numScheduledFlushes
- + ", \"current-memory-component\":"
+ return "{\"dir\" : \"" + fileManager.getBaseDir() + "\", \"memory\" : "
+ + (memoryComponents == null ? 0 : memoryComponents) + ", \"disk\" : " + diskComponents.size()
+ + ", \"num-scheduled-flushes\":" + numScheduledFlushes + ", \"current-memory-component\":"
+ (currentMutableComponentId == null ? "" : currentMutableComponentId.get()) + "}";
}
@@ -914,8 +913,9 @@
if (!memoryComponent.isModified() || opCtx.getOperation() == IndexOperation.DELETE_COMPONENTS) {
return EmptyComponent.INSTANCE;
}
- LOGGER.debug("flushing component with id {} in the index {}",
- ((FlushOperation) operation).getFlushingComponent().getId(), this);
+ FlushOperation flushOperation = (FlushOperation) operation;
+ ILSMComponent component = flushOperation.getFlushingComponent();
+ LOGGER.debug("Flushing {} memory component {}", operation, component);
return doFlush(operation);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
index a42f64b..919dc60 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
@@ -351,9 +351,9 @@
@Override
public String toString() {
- return "{\"class\":\"" + getClass().getSimpleName() + "\", \"state\":\"" + state + "\", \"writers\":"
- + writerCount + ", \"readers\":" + readerCount + ", \"pendingFlushes\":" + pendingFlushes
- + ", \"id\":\"" + componentId + "\", \"index\":" + getIndex() + "}";
+ return "{\"state\":\"" + state + "\", \"writers\":" + writerCount + ", \"readers\":" + readerCount
+ + ", \"pendingFlushes\":" + pendingFlushes + ", \"id\":\"" + componentId + "\", \"index\":" + getIndex()
+ + "}";
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java
index 8e5c982..0214712 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java
@@ -173,7 +173,7 @@
}
private void checkOperation() throws HyracksDataException {
- if (operation.getIOOpertionType() == LSMIOOperationType.MERGE && ++tupleCounter % CHECK_CYCLE == 0) {
+ if (operation.getIOOperationType() == LSMIOOperationType.MERGE && ++tupleCounter % CHECK_CYCLE == 0) {
tupleCounter = 0;
((MergeOperation) operation).waitIfPaused();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
index 260793a..25da568 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
@@ -59,7 +59,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.FLUSH;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java
index 7e6e70b..3dac2c0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java
@@ -73,7 +73,7 @@
if (failed) {
fail(executedOp, t != null ? t : executedOp.getFailure());
}
- if (!failed || executedOp.getIOOpertionType() != LSMIOOperationType.FLUSH) {
+ if (!failed || executedOp.getIOOperationType() != LSMIOOperationType.FLUSH) {
executedOp.complete(); // destroy if merge, cleanup, or successful flush
}
scheduler.completeOperation(executedOp);
@@ -81,7 +81,7 @@
private void fail(ILSMIOOperation executedOp, Throwable t) {
callback.operationFailed(executedOp, t);
- if (executedOp.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (executedOp.getIOOperationType() == LSMIOOperationType.FLUSH) {
executedOp.complete();
// Doesn't make sense to process further flush requests... Mark the operation group permanently failed
// Fail other scheduled operations
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java
index 3eb86b3..2750456 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java
@@ -41,7 +41,7 @@
}
@Override
- public ILSMIOOperation.LSMIOOperationType getIOOpertionType() {
+ public ILSMIOOperation.LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.CLEANUP;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 182887e..263b1fa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -149,7 +149,7 @@
boolean isMutableComponent = numEntered == 0 && component.getType() == LSMComponentType.MEMORY;
if (!component.threadEnter(opType, isMutableComponent)) {
if (dumpState) {
- LOGGER.info("couldn't enter component: {}", component.dumpState());
+ LOGGER.warn("couldn't enter component: {}", component.dumpState());
}
break;
}
@@ -515,7 +515,7 @@
@SuppressWarnings("squid:S2142")
@Override
public void flush(ILSMIOOperation operation) throws HyracksDataException {
- LOGGER.debug("Started a flush operation for index: {}", lsmIndex);
+ LOGGER.debug("Started FLUSH operation {} for index: {}", operation, lsmIndex);
synchronized (opTracker) {
while (!enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.FLUSH, false)) {
try {
@@ -537,14 +537,16 @@
operation.getAccessor().getOpContext().getSearchOperationCallback(),
operation.getAccessor().getOpContext().getModificationCallback());
}
- LOGGER.debug("Finished the flush operation for {}. Result: {}",
- (newComponent == null ? lsmIndex : newComponent), operation.getStatus());
+ String flushedComponent =
+ operation.getStatus() == LSMIOOperationStatus.SUCCESS ? newComponent.toString() : "FAILED";
+ LOGGER.debug("Finished FLUSH operation {} to new disk component {}.", operation, flushedComponent,
+ operation.getStatus());
}
public void doIo(ILSMIOOperation operation) {
try {
operation.getCallback().beforeOperation(operation);
- ILSMDiskComponent newComponent = operation.getIOOpertionType() == LSMIOOperationType.FLUSH
+ ILSMDiskComponent newComponent = operation.getIOOperationType() == LSMIOOperationType.FLUSH
? lsmIndex.flush(operation) : lsmIndex.merge(operation);
operation.setNewComponent(newComponent);
operation.getCallback().afterOperation(operation);
@@ -555,7 +557,7 @@
operation.setStatus(LSMIOOperationStatus.FAILURE);
operation.setFailure(e);
if (LOGGER.isErrorEnabled()) {
- LOGGER.log(Level.ERROR, "{} operation failed on {}", operation.getIOOpertionType(), lsmIndex, e);
+ LOGGER.error("{} operation {} failed on {}", operation.getIOOperationType(), operation, lsmIndex, e);
}
} finally {
try {
@@ -564,8 +566,8 @@
operation.setStatus(LSMIOOperationStatus.FAILURE);
operation.setFailure(th);
if (LOGGER.isErrorEnabled()) {
- LOGGER.log(Level.ERROR, "{} operation.afterFinalize failed on {}", operation.getIOOpertionType(),
- lsmIndex, th);
+ LOGGER.error("{} operation.afterFinalize failed on {}", operation.getIOOperationType(), lsmIndex,
+ th);
}
}
}
@@ -577,11 +579,9 @@
@Override
public void merge(ILSMIOOperation operation) throws HyracksDataException {
- if (LOGGER.isDebugEnabled()) {
- MergeOperation mergeOp = (MergeOperation) operation;
- LOGGER.debug("Started a merge operation (number of merging components {}) for index {}",
- mergeOp.getMergingComponents().size(), lsmIndex);
- }
+ List<ILSMComponent> mergingComponents = ((MergeOperation) operation).getMergingComponents();
+ LOGGER.debug("Started MERGE operation {} of {} components for index: {}", operation, mergingComponents.size(),
+ lsmIndex);
synchronized (opTracker) {
enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE, false);
}
@@ -596,8 +596,9 @@
operation.getAccessor().getOpContext().getSearchOperationCallback(),
operation.getAccessor().getOpContext().getModificationCallback());
}
- LOGGER.debug("Finished the merge operation for {}. Result: {}",
- (newComponent == null ? lsmIndex : newComponent), operation.getStatus());
+ String mergedComponent =
+ operation.getStatus() == LSMIOOperationStatus.SUCCESS ? newComponent.toString() : "FAILED";
+ LOGGER.debug("Finished MERGE operation {}. New component: {}", operation, mergedComponent);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java
index 170076c..bcb5dd9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java
@@ -36,7 +36,7 @@
}
@Override
- public final LSMIOOperationType getIOOpertionType() {
+ public final LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.LOAD;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
index 74f8913..6b0d572 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
@@ -54,7 +54,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.MERGE;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
index 036ade2..edf89c9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
@@ -54,7 +54,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.NOOP;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
index e6fb3b1..d2a1cbf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
@@ -75,7 +75,7 @@
}
private void after(ILSMIOOperation operation) {
- if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (operation.getIOOperationType() == LSMIOOperationType.FLUSH) {
synchronized (runningFlushOperations) {
runningFlushOperations.remove(operation.getIndexIdentifier());
if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
@@ -91,7 +91,7 @@
private void before(ILSMIOOperation operation) throws InterruptedException {
String id = operation.getIndexIdentifier();
- if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (operation.getIOOperationType() == LSMIOOperationType.FLUSH) {
synchronized (runningFlushOperations) {
while (true) {
if (failedGroups.containsKey(id)) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
index eb48f82..b9719ea 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
@@ -34,13 +34,8 @@
import org.apache.hyracks.util.trace.ITracer;
import org.apache.hyracks.util.trace.ITracer.Scope;
import org.apache.hyracks.util.trace.TraceUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
class TracedIOOperation implements ILSMIOOperation {
-
- static final Logger LOGGER = LogManager.getLogger();
-
protected final ILSMIOOperation ioOp;
private final LSMIOOperationType ioOpType;
private final ITracer tracer;
@@ -49,12 +44,12 @@
protected TracedIOOperation(ILSMIOOperation ioOp, ITracer tracer, long traceCategory) {
this.ioOp = ioOp;
this.tracer = tracer;
- this.ioOpType = ioOp.getIOOpertionType();
+ this.ioOpType = ioOp.getIOOperationType();
this.traceCategory = traceCategory;
}
public static ILSMIOOperation wrap(final ILSMIOOperation ioOp, final ITracer tracer) {
- final String ioOpName = ioOp.getIOOpertionType().name().toLowerCase();
+ final String ioOpName = ioOp.getIOOperationType().name().toLowerCase();
final long traceCategory = tracer.getRegistry().get(TraceUtils.INDEX_IO_OPERATIONS);
if (tracer.isEnabled(traceCategory)) {
tracer.instant("schedule-" + ioOpName, traceCategory, Scope.p,
@@ -84,7 +79,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return ioOpType;
}
@@ -95,7 +90,7 @@
try {
return ioOp.call();
} finally {
- tracer.durationE(ioOp.getIOOpertionType().name().toLowerCase(), traceCategory, tid, () -> "{\"size\":"
+ tracer.durationE(ioOp.getIOOperationType().name().toLowerCase(), traceCategory, tid, () -> "{\"size\":"
+ getTarget().getFile().length() + ", \"path\": \"" + ioOp.getTarget().getRelativePath() + "\"}");
}
}
@@ -199,4 +194,9 @@
public boolean isCompleted() {
return ioOp.isCompleted();
}
+
+ @Override
+ public String toString() {
+ return ioOp.toString();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
index a298fe5..e7ac63c 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
@@ -394,7 +394,7 @@
}
public static void checkCursorStats(ILSMIOOperation op) {
- if (op.getIOOpertionType() == LSMIOOperationType.MERGE) {
+ if (op.getIOOperationType() == LSMIOOperationType.MERGE) {
MergeOperation mergeOp = (MergeOperation) op;
IIndexCursorStats stats = mergeOp.getCursorStats();
Assert.assertTrue(stats.getPageCounter().get() > 0);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
index 100139a..a14d4ec 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
@@ -158,7 +158,7 @@
ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
ILSMIOOperation mergeOp = accessor.scheduleMerge(((LSMBTree) ctx.getIndex()).getDiskComponents());
mergeOp.addCompleteListener(op -> {
- if (op.getIOOpertionType() == LSMIOOperationType.MERGE) {
+ if (op.getIOOperationType() == LSMIOOperationType.MERGE) {
long numPages = op.getNewComponent().getComponentSize()
/ harness.getDiskBufferCache().getPageSizeWithHeader() - 1;
// we skipped the metadata page for simplicity
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java
index 15f65a4..10bcb60 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java
@@ -28,7 +28,6 @@
import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
import org.apache.hyracks.storage.am.lsm.common.impls.GreedyScheduler;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIoOperationFailedCallback;
-import org.apache.hyracks.storage.am.lsm.common.test.IoSchedulerTest.MockedOperation;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
@@ -221,7 +220,7 @@
ILSMIOOperation op = Mockito.mock(ILSMIOOperation.class);
MockedOperation mockedOp = new MockedOperation(op);
Mockito.when(op.getIndexIdentifier()).thenReturn(index);
- Mockito.when(op.getIOOpertionType()).thenReturn(type);
+ Mockito.when(op.getIOOperationType()).thenReturn(type);
Mockito.when(op.getRemainingPages()).thenReturn(remainingPages);
Mockito.doAnswer(new Answer<Boolean>() {