[ASTERIXDB-3428][OTH] Improve storage logging
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Storage log events will be in storage_debug.log
- Reduce schema logging on query
- Several improvements when logging on merge/flush
Change-Id: I8b4e92585dee55a7f6a24342fb8d1a6390449c1c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18368
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
index 0253b65..7010e88 100644
--- a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
+++ b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
@@ -62,6 +62,10 @@
LOGGER.info("Used space: {}, pressureCapacity: {} (isPressured: {})",
StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
true);
+ } else if (usedSpace >= diskSpace.getAllocatedCapacity()) {
+ LOGGER.warn("Allocated disk space has been exceeded. Used space: {}, pressureCapacity: {}",
+ StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
+ true);
} else {
LOGGER.debug("Used space: {}, pressureCapacity: {} (isPressured: {})",
StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 3a0f2b9..5e241c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -237,6 +237,6 @@
@Override
public String toString() {
- return "{\"class\":\"" + getClass().getSimpleName() + "\",\"file\":\"" + file.getRelativePath() + "\"}";
+ return "{\"class\": \"" + getClass().getSimpleName() + "\", \"file\": \"" + file.getRelativePath() + "\"}";
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
index 8d3ae32..6dde7e9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeWithBloomFilterDiskComponent.java
@@ -72,7 +72,7 @@
private IChainedComponentBulkLoader createColumnIndexBulkLoader(ILSMIOOperation operation, float fillFactor,
boolean verifyInput, IPageWriteCallback callback) throws HyracksDataException {
- LSMIOOperationType operationType = operation.getIOOpertionType();
+ LSMIOOperationType operationType = operation.getIOOperationType();
LSMColumnBTree lsmColumnBTree = (LSMColumnBTree) getLsmIndex();
ColumnBTree columnBTree = (ColumnBTree) getIndex();
IColumnMetadata columnMetadata;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
index b3da5f2..958923f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
@@ -91,5 +91,9 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index e2e49e3..705701e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -38,9 +38,11 @@
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
-
+ private static final Logger LOGGER = LogManager.getLogger();
private static final FilenameFilter BTREE_FILTER =
(dir, name) -> !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
@@ -97,6 +99,8 @@
validateFiles(btreeFilesSet, allBloomFilterFiles, BLOOM_FILTER_FILTER, null, btreeFactory.getBufferCache());
// Sanity check.
if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
+ LOGGER.error("Unequal number of trees and filters. Trees: {}, Filters: {}", allBTreeFiles,
+ allBloomFilterFiles);
throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
index 822df6f..278b90f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
@@ -74,7 +74,7 @@
/**
* @return the operation type
*/
- LSMIOOperationType getIOOpertionType();
+ LSMIOOperationType getIOOperationType();
@Override
LSMIOOperationStatus call() throws HyracksDataException;
@@ -92,8 +92,7 @@
/**
* clean up left over files in case of an exception during execution
*
- * @param bufferCache
- * a buffercache that manages the files
+ * @param bufferCache a buffer cache that manages the files
*/
void cleanup(IBufferCache bufferCache);
@@ -157,7 +156,6 @@
Map<String, Object> getParameters();
/**
- *
* @return the estimated number of disk pages remaining for this IO operation
*/
long getRemainingPages();
@@ -173,7 +171,6 @@
void pause();
/**
- *
* @return whether this IO operation is currently active (i.e., not paused)
*/
boolean isActive();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java
index 3a8e558..602ea14 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java
@@ -52,7 +52,7 @@
@Override
public void scheduleOperation(ILSMIOOperation operation) {
- switch (operation.getIOOpertionType()) {
+ switch (operation.getIOOperationType()) {
case FLUSH:
scheduleFlush(operation);
break;
@@ -70,13 +70,13 @@
default:
// this should never happen
// just guard here to avoid silent failures in case of future extensions
- throw new IllegalArgumentException("Unknown operation type " + operation.getIOOpertionType());
+ throw new IllegalArgumentException("Unknown operation type " + operation.getIOOperationType());
}
}
@Override
public void completeOperation(ILSMIOOperation operation) throws HyracksDataException {
- switch (operation.getIOOpertionType()) {
+ switch (operation.getIOOperationType()) {
case FLUSH:
completeFlush(operation);
break;
@@ -91,7 +91,7 @@
default:
// this should never happen
// just guard here to avoid silent failures in case of future extensions
- throw new IllegalArgumentException("Unknown operation type " + operation.getIOOpertionType());
+ throw new IllegalArgumentException("Unknown operation type " + operation.getIOOperationType());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
index a5f70d9..8ae0571 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
@@ -219,4 +219,10 @@
}
}
}
+
+ @Override
+ public String toString() {
+ String fileName = target != null ? "\"fileName\": \"" + target.getFile().getName() + "\", " : "";
+ return "{" + fileName + "\"ioOpID\": " + System.identityHashCode(this) + "}";
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
index 966d6d1..baf0931 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
@@ -153,8 +153,7 @@
/**
* Mark the component as valid
*
- * @param persist
- * whether the call should force data to disk before returning
+ * @param persist whether the call should force data to disk before returning
* @throws HyracksDataException
*/
@Override
@@ -234,7 +233,7 @@
if (withFilter && getLsmIndex().getFilterFields() != null) {
chainedBulkLoader.addBulkLoader(createFilterBulkLoader());
}
- IChainedComponentBulkLoader indexBulkloader = operation.getIOOpertionType() == LSMIOOperationType.MERGE
+ IChainedComponentBulkLoader indexBulkloader = operation.getIOOperationType() == LSMIOOperationType.MERGE
? createMergeIndexBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, callback)
: createIndexBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, callback);
chainedBulkLoader.addBulkLoader(indexBulkloader);
@@ -245,7 +244,6 @@
@Override
public String toString() {
- return "{\"class\":" + getClass().getSimpleName() + "\", \"id\":" + componentId + ", \"index\":" + getIndex()
- + "}";
+ return "{\"id\": " + componentId + ", index:" + getIndex() + "}";
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index f7a15fb..25d928c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -728,10 +728,9 @@
@Override
public final String toString() {
- return "{\"class\" : \"" + getClass().getSimpleName() + "\", \"dir\" : \"" + fileManager.getBaseDir()
- + "\", \"memory\" : " + (memoryComponents == null ? 0 : memoryComponents) + ", \"disk\" : "
- + diskComponents.size() + ", \"num-scheduled-flushes\":" + numScheduledFlushes
- + ", \"current-memory-component\":"
+ return "{\"dir\" : \"" + fileManager.getBaseDir() + "\", \"memory\" : "
+ + (memoryComponents == null ? 0 : memoryComponents) + ", \"disk\" : " + diskComponents.size()
+ + ", \"num-scheduled-flushes\":" + numScheduledFlushes + ", \"current-memory-component\":"
+ (currentMutableComponentId == null ? "" : currentMutableComponentId.get()) + "}";
}
@@ -914,8 +913,9 @@
if (!memoryComponent.isModified() || opCtx.getOperation() == IndexOperation.DELETE_COMPONENTS) {
return EmptyComponent.INSTANCE;
}
- LOGGER.debug("flushing component with id {} in the index {}",
- ((FlushOperation) operation).getFlushingComponent().getId(), this);
+ FlushOperation flushOperation = (FlushOperation) operation;
+ ILSMComponent component = flushOperation.getFlushingComponent();
+ LOGGER.debug("Flushing {} memory component {}", operation, component);
return doFlush(operation);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
index a42f64b..919dc60 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
@@ -351,9 +351,9 @@
@Override
public String toString() {
- return "{\"class\":\"" + getClass().getSimpleName() + "\", \"state\":\"" + state + "\", \"writers\":"
- + writerCount + ", \"readers\":" + readerCount + ", \"pendingFlushes\":" + pendingFlushes
- + ", \"id\":\"" + componentId + "\", \"index\":" + getIndex() + "}";
+ return "{\"state\":\"" + state + "\", \"writers\":" + writerCount + ", \"readers\":" + readerCount
+ + ", \"pendingFlushes\":" + pendingFlushes + ", \"id\":\"" + componentId + "\", \"index\":" + getIndex()
+ + "}";
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java
index 8e5c982..0214712 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java
@@ -173,7 +173,7 @@
}
private void checkOperation() throws HyracksDataException {
- if (operation.getIOOpertionType() == LSMIOOperationType.MERGE && ++tupleCounter % CHECK_CYCLE == 0) {
+ if (operation.getIOOperationType() == LSMIOOperationType.MERGE && ++tupleCounter % CHECK_CYCLE == 0) {
tupleCounter = 0;
((MergeOperation) operation).waitIfPaused();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
index 260793a..25da568 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
@@ -59,7 +59,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.FLUSH;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java
index 7e6e70b..3dac2c0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOperationExecutor.java
@@ -73,7 +73,7 @@
if (failed) {
fail(executedOp, t != null ? t : executedOp.getFailure());
}
- if (!failed || executedOp.getIOOpertionType() != LSMIOOperationType.FLUSH) {
+ if (!failed || executedOp.getIOOperationType() != LSMIOOperationType.FLUSH) {
executedOp.complete(); // destroy if merge, cleanup, or successful flush
}
scheduler.completeOperation(executedOp);
@@ -81,7 +81,7 @@
private void fail(ILSMIOOperation executedOp, Throwable t) {
callback.operationFailed(executedOp, t);
- if (executedOp.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (executedOp.getIOOperationType() == LSMIOOperationType.FLUSH) {
executedOp.complete();
// Doesn't make sense to process further flush requests... Mark the operation group permanently failed
// Fail other scheduled operations
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java
index 3eb86b3..2750456 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMCleanupOperation.java
@@ -41,7 +41,7 @@
}
@Override
- public ILSMIOOperation.LSMIOOperationType getIOOpertionType() {
+ public ILSMIOOperation.LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.CLEANUP;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 182887e..263b1fa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -149,7 +149,7 @@
boolean isMutableComponent = numEntered == 0 && component.getType() == LSMComponentType.MEMORY;
if (!component.threadEnter(opType, isMutableComponent)) {
if (dumpState) {
- LOGGER.info("couldn't enter component: {}", component.dumpState());
+ LOGGER.warn("couldn't enter component: {}", component.dumpState());
}
break;
}
@@ -515,7 +515,7 @@
@SuppressWarnings("squid:S2142")
@Override
public void flush(ILSMIOOperation operation) throws HyracksDataException {
- LOGGER.debug("Started a flush operation for index: {}", lsmIndex);
+ LOGGER.debug("Started FLUSH operation {} for index: {}", operation, lsmIndex);
synchronized (opTracker) {
while (!enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.FLUSH, false)) {
try {
@@ -537,14 +537,16 @@
operation.getAccessor().getOpContext().getSearchOperationCallback(),
operation.getAccessor().getOpContext().getModificationCallback());
}
- LOGGER.debug("Finished the flush operation for {}. Result: {}",
- (newComponent == null ? lsmIndex : newComponent), operation.getStatus());
+ String flushedComponent =
+ operation.getStatus() == LSMIOOperationStatus.SUCCESS ? newComponent.toString() : "FAILED";
+ LOGGER.debug("Finished FLUSH operation {} to new disk component {}.", operation, flushedComponent,
+ operation.getStatus());
}
public void doIo(ILSMIOOperation operation) {
try {
operation.getCallback().beforeOperation(operation);
- ILSMDiskComponent newComponent = operation.getIOOpertionType() == LSMIOOperationType.FLUSH
+ ILSMDiskComponent newComponent = operation.getIOOperationType() == LSMIOOperationType.FLUSH
? lsmIndex.flush(operation) : lsmIndex.merge(operation);
operation.setNewComponent(newComponent);
operation.getCallback().afterOperation(operation);
@@ -555,7 +557,7 @@
operation.setStatus(LSMIOOperationStatus.FAILURE);
operation.setFailure(e);
if (LOGGER.isErrorEnabled()) {
- LOGGER.log(Level.ERROR, "{} operation failed on {}", operation.getIOOpertionType(), lsmIndex, e);
+ LOGGER.error("{} operation {} failed on {}", operation.getIOOperationType(), operation, lsmIndex, e);
}
} finally {
try {
@@ -564,8 +566,8 @@
operation.setStatus(LSMIOOperationStatus.FAILURE);
operation.setFailure(th);
if (LOGGER.isErrorEnabled()) {
- LOGGER.log(Level.ERROR, "{} operation.afterFinalize failed on {}", operation.getIOOpertionType(),
- lsmIndex, th);
+ LOGGER.error("{} operation.afterFinalize failed on {}", operation.getIOOperationType(), lsmIndex,
+ th);
}
}
}
@@ -577,11 +579,9 @@
@Override
public void merge(ILSMIOOperation operation) throws HyracksDataException {
- if (LOGGER.isDebugEnabled()) {
- MergeOperation mergeOp = (MergeOperation) operation;
- LOGGER.debug("Started a merge operation (number of merging components {}) for index {}",
- mergeOp.getMergingComponents().size(), lsmIndex);
- }
+ List<ILSMComponent> mergingComponents = ((MergeOperation) operation).getMergingComponents();
+ LOGGER.debug("Started MERGE operation {} of {} components for index: {}", operation, mergingComponents.size(),
+ lsmIndex);
synchronized (opTracker) {
enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE, false);
}
@@ -596,8 +596,9 @@
operation.getAccessor().getOpContext().getSearchOperationCallback(),
operation.getAccessor().getOpContext().getModificationCallback());
}
- LOGGER.debug("Finished the merge operation for {}. Result: {}",
- (newComponent == null ? lsmIndex : newComponent), operation.getStatus());
+ String mergedComponent =
+ operation.getStatus() == LSMIOOperationStatus.SUCCESS ? newComponent.toString() : "FAILED";
+ LOGGER.debug("Finished MERGE operation {}. New component: {}", operation, mergedComponent);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java
index 170076c..bcb5dd9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LoadOperation.java
@@ -36,7 +36,7 @@
}
@Override
- public final LSMIOOperationType getIOOpertionType() {
+ public final LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.LOAD;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
index 74f8913..6b0d572 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
@@ -54,7 +54,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.MERGE;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
index 036ade2..edf89c9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
@@ -54,7 +54,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return LSMIOOperationType.NOOP;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
index e6fb3b1..d2a1cbf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
@@ -75,7 +75,7 @@
}
private void after(ILSMIOOperation operation) {
- if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (operation.getIOOperationType() == LSMIOOperationType.FLUSH) {
synchronized (runningFlushOperations) {
runningFlushOperations.remove(operation.getIndexIdentifier());
if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
@@ -91,7 +91,7 @@
private void before(ILSMIOOperation operation) throws InterruptedException {
String id = operation.getIndexIdentifier();
- if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ if (operation.getIOOperationType() == LSMIOOperationType.FLUSH) {
synchronized (runningFlushOperations) {
while (true) {
if (failedGroups.containsKey(id)) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
index eb48f82..b9719ea 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
@@ -34,13 +34,8 @@
import org.apache.hyracks.util.trace.ITracer;
import org.apache.hyracks.util.trace.ITracer.Scope;
import org.apache.hyracks.util.trace.TraceUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
class TracedIOOperation implements ILSMIOOperation {
-
- static final Logger LOGGER = LogManager.getLogger();
-
protected final ILSMIOOperation ioOp;
private final LSMIOOperationType ioOpType;
private final ITracer tracer;
@@ -49,12 +44,12 @@
protected TracedIOOperation(ILSMIOOperation ioOp, ITracer tracer, long traceCategory) {
this.ioOp = ioOp;
this.tracer = tracer;
- this.ioOpType = ioOp.getIOOpertionType();
+ this.ioOpType = ioOp.getIOOperationType();
this.traceCategory = traceCategory;
}
public static ILSMIOOperation wrap(final ILSMIOOperation ioOp, final ITracer tracer) {
- final String ioOpName = ioOp.getIOOpertionType().name().toLowerCase();
+ final String ioOpName = ioOp.getIOOperationType().name().toLowerCase();
final long traceCategory = tracer.getRegistry().get(TraceUtils.INDEX_IO_OPERATIONS);
if (tracer.isEnabled(traceCategory)) {
tracer.instant("schedule-" + ioOpName, traceCategory, Scope.p,
@@ -84,7 +79,7 @@
}
@Override
- public LSMIOOperationType getIOOpertionType() {
+ public LSMIOOperationType getIOOperationType() {
return ioOpType;
}
@@ -95,7 +90,7 @@
try {
return ioOp.call();
} finally {
- tracer.durationE(ioOp.getIOOpertionType().name().toLowerCase(), traceCategory, tid, () -> "{\"size\":"
+ tracer.durationE(ioOp.getIOOperationType().name().toLowerCase(), traceCategory, tid, () -> "{\"size\":"
+ getTarget().getFile().length() + ", \"path\": \"" + ioOp.getTarget().getRelativePath() + "\"}");
}
}
@@ -199,4 +194,9 @@
public boolean isCompleted() {
return ioOp.isCompleted();
}
+
+ @Override
+ public String toString() {
+ return ioOp.toString();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
index a298fe5..e7ac63c 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
@@ -394,7 +394,7 @@
}
public static void checkCursorStats(ILSMIOOperation op) {
- if (op.getIOOpertionType() == LSMIOOperationType.MERGE) {
+ if (op.getIOOperationType() == LSMIOOperationType.MERGE) {
MergeOperation mergeOp = (MergeOperation) op;
IIndexCursorStats stats = mergeOp.getCursorStats();
Assert.assertTrue(stats.getPageCounter().get() > 0);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
index 100139a..a14d4ec 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
@@ -158,7 +158,7 @@
ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
ILSMIOOperation mergeOp = accessor.scheduleMerge(((LSMBTree) ctx.getIndex()).getDiskComponents());
mergeOp.addCompleteListener(op -> {
- if (op.getIOOpertionType() == LSMIOOperationType.MERGE) {
+ if (op.getIOOperationType() == LSMIOOperationType.MERGE) {
long numPages = op.getNewComponent().getComponentSize()
/ harness.getDiskBufferCache().getPageSizeWithHeader() - 1;
// we skipped the metadata page for simplicity
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java
index 15f65a4..10bcb60 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/IoSchedulerTest.java
@@ -28,7 +28,6 @@
import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
import org.apache.hyracks.storage.am.lsm.common.impls.GreedyScheduler;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIoOperationFailedCallback;
-import org.apache.hyracks.storage.am.lsm.common.test.IoSchedulerTest.MockedOperation;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
@@ -221,7 +220,7 @@
ILSMIOOperation op = Mockito.mock(ILSMIOOperation.class);
MockedOperation mockedOp = new MockedOperation(op);
Mockito.when(op.getIndexIdentifier()).thenReturn(index);
- Mockito.when(op.getIOOpertionType()).thenReturn(type);
+ Mockito.when(op.getIOOperationType()).thenReturn(type);
Mockito.when(op.getRemainingPages()).thenReturn(remainingPages);
Mockito.doAnswer(new Answer<Boolean>() {