reduced log record size by removing old operation/value and generalized undo operation by not using physical delete operation in LSM btree
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILogRecord.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILogRecord.java
index 3068867..b9baa3b 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILogRecord.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILogRecord.java
@@ -22,7 +22,7 @@
public static final int JOB_TERMINATE_LOG_SIZE = 13; //JOB_COMMIT or ABORT log type
public static final int ENTITY_COMMIT_LOG_BASE_SIZE = 25;
- public static final int UPDATE_LOG_BASE_SIZE = 60;
+ public static final int UPDATE_LOG_BASE_SIZE = 54;
public boolean readLogRecord(ByteBuffer buffer);
@@ -65,10 +65,6 @@
public void setResourceId(long resourceId);
- public byte getResourceType();
-
- public void setResourceType(byte resourceType);
-
public int getLogSize();
public void setLogSize(int logSize);
@@ -85,18 +81,6 @@
public void setNewValue(ITupleReference newValue);
- public byte getOldOp();
-
- public void setOldOp(byte oldOp);
-
- public int getOldValueSize();
-
- public void setOldValueSize(int oldValueSize);
-
- public ITupleReference getOldValue();
-
- public void setOldValue(ITupleReference oldValue);
-
public long getChecksum();
public void setChecksum(long checksum);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/AbstractIndexModificationOperationCallback.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/AbstractIndexModificationOperationCallback.java
index 031a26e..cba690d 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/AbstractIndexModificationOperationCallback.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/AbstractIndexModificationOperationCallback.java
@@ -51,11 +51,10 @@
logRecord.setJobId(txnCtx.getJobId().getId());
logRecord.setDatasetId(datasetId);
logRecord.setResourceId(resourceId);
- logRecord.setResourceType(resourceType);
logRecord.setNewOp((byte) (indexOp.ordinal()));
}
- protected void log(int PKHash, ITupleReference newValue, IndexOperation oldOp, ITupleReference oldValue)
+ protected void log(int PKHash, ITupleReference newValue)
throws ACIDException {
logRecord.setPKHashValue(PKHash);
logRecord.setPKFields(primaryKeyFields);
@@ -67,15 +66,6 @@
} else {
logRecord.setNewValueSize(0);
}
- if (resourceType == ResourceType.LSM_BTREE) {
- logRecord.setOldOp((byte) (oldOp.ordinal()));
- if (oldValue != null) {
- logRecord.setOldValueSize(tupleWriter.bytesRequired(oldValue));
- logRecord.setOldValue(oldValue);
- } else {
- logRecord.setOldValueSize(0);
- }
- }
logRecord.computeAndSetLogSize();
txnSubsystem.getLogManager().log(logRecord);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java
index 44dcad8..4dc4a10 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
/**
* Assumes LSM-BTrees as primary indexes.
@@ -53,15 +52,7 @@
public void found(ITupleReference before, ITupleReference after) throws HyracksDataException {
try {
int pkHash = computePrimaryKeyHashValue(after, primaryKeyFields);
- LSMBTreeTupleReference lsmBTreeTuple = (LSMBTreeTupleReference) before;
- IndexOperation oldOp = IndexOperation.INSERT;
- if (before == null) {
- oldOp = IndexOperation.NOOP;
- }
- if (lsmBTreeTuple != null && lsmBTreeTuple.isAntimatter()) {
- oldOp = IndexOperation.DELETE;
- }
- log(pkHash, after, oldOp, before);
+ log(pkHash, after);
} catch (ACIDException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java
index ef5ad21..e60501a 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java
@@ -17,14 +17,12 @@
import edu.uci.ics.asterix.common.exceptions.ACIDException;
import edu.uci.ics.asterix.common.transactions.ILockManager;
-import edu.uci.ics.asterix.common.transactions.IRecoveryManager.ResourceType;
import edu.uci.ics.asterix.common.transactions.ITransactionContext;
import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
/**
* Secondary-index modifications do not require any locking.
@@ -52,20 +50,7 @@
public void found(ITupleReference before, ITupleReference after) throws HyracksDataException {
try {
int pkHash = computePrimaryKeyHashValue(after, primaryKeyFields);
- IndexOperation effectiveOldOp;
- if (resourceType == ResourceType.LSM_BTREE) {
- LSMBTreeTupleReference lsmBTreeTuple = (LSMBTreeTupleReference) before;
- if (before == null) {
- effectiveOldOp = IndexOperation.NOOP;
- } else if (lsmBTreeTuple != null && lsmBTreeTuple.isAntimatter()) {
- effectiveOldOp = IndexOperation.DELETE;
- } else {
- effectiveOldOp = IndexOperation.INSERT;
- }
- } else {
- effectiveOldOp = oldOp;
- }
- this.log(pkHash, after, effectiveOldOp, before);
+ this.log(pkHash, after);
} catch (ACIDException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecord.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecord.java
index dd81df7..1337e33 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecord.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecord.java
@@ -40,10 +40,9 @@
* PKValueSize(4)
* PKValue(PKValueSize)
* ---------------------------
- * [Header3] (21 bytes) : only for update log type
+ * [Header3] (20 bytes) : only for update log type
* PrevLSN(8)
* ResourceId(8) //stored in .metadata of the corresponding index in NC node
- * ResourceType(1)
* LogRecordSize(4)
* ---------------------------
* [Body] (Variable size) : only for update log type
@@ -51,9 +50,6 @@
* NewOp(1)
* NewValueSize(4)
* NewValue(NewValueSize)
- * OldOp(1)
- * OldValueSize(4)
- * OldValue(OldValueSize)
* ---------------------------
* [Tail] (8 bytes) : for all log types
* Checksum(8)
@@ -62,8 +58,8 @@
* 1) JOB_COMMIT_LOG_SIZE: 13 bytes (5 + 8)
* 2) ENTITY_COMMIT: 25 + PKSize (5 + 12 + PKSize + 8)
* --> ENTITY_COMMIT_LOG_BASE_SIZE = 25
- * 3) UPDATE: 64 + PKSize + New/OldValueSize (5 + 12 + PKSize + 21 + 14 + New/OldValueSize + 8)
- * --> UPDATE_LOG_BASE_SIZE = 60
+ * 3) UPDATE: 54 + PKValueSize + NewValueSize (5 + 12 + PKValueSize + 20 + 9 + NewValueSize + 8)
+ * --> UPDATE_LOG_BASE_SIZE = 54
*/
public class LogRecord implements ILogRecord {
@@ -76,15 +72,11 @@
private ITupleReference PKValue;
private long prevLSN;
private long resourceId;
- private byte resourceType;
private int logSize;
private int fieldCnt;
private byte newOp;
private int newValueSize;
private ITupleReference newValue;
- private byte oldOp;
- private int oldValueSize;
- private ITupleReference oldValue;
private long checksum;
//------------- fields in a log record (end) --------------//
@@ -96,7 +88,6 @@
private final SimpleTupleWriter tupleWriter;
private final PrimaryKeyTupleReference readPKValue;
private final SimpleTupleReference readNewValue;
- private final SimpleTupleReference readOldValue;
private final CRC32 checksumGen;
private int[] PKFields;
@@ -105,7 +96,6 @@
tupleWriter = new SimpleTupleWriter();
readPKValue = new PrimaryKeyTupleReference();
readNewValue = (SimpleTupleReference) tupleWriter.createTupleReference();
- readOldValue = (SimpleTupleReference) tupleWriter.createTupleReference();
checksumGen = new CRC32();
}
@@ -126,21 +116,11 @@
if (logType == LogType.UPDATE) {
buffer.putLong(prevLSN);
buffer.putLong(resourceId);
- buffer.put(resourceType);
buffer.putInt(logSize);
buffer.putInt(fieldCnt);
buffer.put(newOp);
buffer.putInt(newValueSize);
writeTuple(buffer, newValue, newValueSize);
- if (resourceType == ResourceType.LSM_BTREE) {
- buffer.put(oldOp);
- if (oldOp != (byte) (IndexOperation.NOOP.ordinal())) {
- buffer.putInt(oldValueSize);
- if (oldValueSize > 0) {
- writeTuple(buffer, oldValue, oldValueSize);
- }
- }
- }
}
checksum = generateChecksum(buffer, beginOffset, logSize - CHECKSUM_SIZE);
buffer.putLong(checksum);
@@ -186,21 +166,11 @@
if (logType == LogType.UPDATE) {
prevLSN = buffer.getLong();
resourceId = buffer.getLong();
- resourceType = buffer.get();
logSize = buffer.getInt();
fieldCnt = buffer.getInt();
newOp = buffer.get();
newValueSize = buffer.getInt();
newValue = readTuple(buffer, readNewValue, fieldCnt, newValueSize);
- if (resourceType == ResourceType.LSM_BTREE) {
- oldOp = buffer.get();
- if (oldOp != (byte) (IndexOperation.NOOP.ordinal())) {
- oldValueSize = buffer.getInt();
- if (oldValueSize > 0) {
- oldValue = readTuple(buffer, readOldValue, fieldCnt, oldValueSize);
- }
- }
- }
} else {
computeAndSetLogSize();
}
@@ -269,14 +239,7 @@
}
private void setUpdateLogSize() {
- logSize = UPDATE_LOG_BASE_SIZE + PKValueSize + newValueSize + oldValueSize;
- if (resourceType != ResourceType.LSM_BTREE) {
- logSize -= 5; //oldOp(byte: 1) + oldValueLength(int: 4)
- } else {
- if (oldOp == (byte) (IndexOperation.NOOP.ordinal())) {
- logSize -= 4; //oldValueLength(int: 4)
- }
- }
+ logSize = UPDATE_LOG_BASE_SIZE + PKValueSize + newValueSize;
}
@Override
@@ -313,7 +276,6 @@
if (logType == LogType.UPDATE) {
builder.append(" PrevLSN : ").append(prevLSN);
builder.append(" ResourceId : ").append(resourceId);
- builder.append(" ResourceType : ").append(resourceType);
}
return builder.toString();
}
@@ -403,16 +365,6 @@
}
@Override
- public byte getResourceType() {
- return resourceType;
- }
-
- @Override
- public void setResourceType(byte resourceType) {
- this.resourceType = resourceType;
- }
-
- @Override
public int getLogSize() {
return logSize;
}
@@ -454,36 +406,6 @@
}
@Override
- public byte getOldOp() {
- return oldOp;
- }
-
- @Override
- public void setOldOp(byte oldOp) {
- this.oldOp = oldOp;
- }
-
- @Override
- public int getOldValueSize() {
- return oldValueSize;
- }
-
- @Override
- public void setOldValueSize(int oldValueSize) {
- this.oldValueSize = oldValueSize;
- }
-
- @Override
- public ITupleReference getOldValue() {
- return oldValue;
- }
-
- @Override
- public void setOldValue(ITupleReference oldValue) {
- this.oldValue = oldValue;
- }
-
- @Override
public long getChecksum() {
return checksum;
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
index 1e34df6..eca2c33 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
@@ -697,22 +697,13 @@
.getIndex(logRecord.getResourceId());
ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
- if (logRecord.getResourceType() == ResourceType.LSM_BTREE) {
- if (logRecord.getOldOp() != IndexOperation.NOOP.ordinal()) {
- if (logRecord.getOldOp() == IndexOperation.DELETE.ordinal()) {
- indexAccessor.forceDelete(logRecord.getOldValue());
- } else {
- indexAccessor.forceInsert(logRecord.getOldValue());
- }
- } else {
- indexAccessor.forcePhysicalDelete(logRecord.getNewValue());
- }
+
+ if (logRecord.getNewOp() == IndexOperation.INSERT.ordinal()) {
+ indexAccessor.forceDelete(logRecord.getNewValue());
+ } else if (logRecord.getNewOp() == IndexOperation.DELETE.ordinal()) {
+ indexAccessor.forceInsert(logRecord.getNewValue());
} else {
- if (logRecord.getNewOp() == IndexOperation.DELETE.ordinal()) {
- indexAccessor.forceInsert(logRecord.getNewValue());
- } else {
- indexAccessor.forceDelete(logRecord.getNewValue());
- }
+ throw new IllegalStateException("Unsupported OperationType: " + logRecord.getNewOp());
}
} catch (Exception e) {
throw new IllegalStateException("Failed to undo", e);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionSubsystem.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionSubsystem.java
index e6bcc5d..9a05e77 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionSubsystem.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionSubsystem.java
@@ -22,7 +22,7 @@
import edu.uci.ics.asterix.common.transactions.IRecoveryManager;
import edu.uci.ics.asterix.common.transactions.ITransactionManager;
import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
-import edu.uci.ics.asterix.transaction.management.service.locking.ConcurrentLockManager;
+import edu.uci.ics.asterix.transaction.management.service.locking.LockManager;
import edu.uci.ics.asterix.transaction.management.service.logging.LogManager;
import edu.uci.ics.asterix.transaction.management.service.recovery.CheckpointThread;
import edu.uci.ics.asterix.transaction.management.service.recovery.RecoveryManager;
@@ -47,7 +47,8 @@
this.id = id;
this.txnProperties = txnProperties;
this.transactionManager = new TransactionManager(this);
- this.lockManager = new ConcurrentLockManager(this);
+ //ConcurrentLockManager should be used when the issue 695 is fixed.
+ this.lockManager = new LockManager(this);
this.logManager = new LogManager(this);
this.recoveryManager = new RecoveryManager(this);
if (asterixAppRuntimeContextProvider != null) {