Merge commit 'fab68d291e' from 'stabilization-f69489'
Change-Id: Idcabca9b2b126f891a12e60e4b427fdc4828f07f
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
index 8f40b3a..3da845d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
@@ -54,6 +54,7 @@
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.operators.LSMIndexBulkLoadOperatorNodePushable;
+import org.apache.asterix.test.base.TestMethodTracer;
import org.apache.asterix.test.common.TestHelper;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -79,7 +80,9 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestRule;
import org.junit.runners.Parameterized;
public class CheckpointInSecondaryIndexTest {
@@ -101,7 +104,7 @@
private static final boolean[] UNIQUE_META_FIELDS = null;
private static final int[] KEY_INDEXES = { 0 };
private static final int[] KEY_INDICATORS = { Index.RECORD_INDICATOR };
- private static final List<Integer> KEY_INDICATORS_LIST = Arrays.asList(new Integer[] { Index.RECORD_INDICATOR });
+ private static final List<Integer> KEY_INDICATORS_LIST = Collections.singletonList(Index.RECORD_INDICATOR);
private static final int RECORDS_PER_COMPONENT = 500;
private static final int DATASET_ID = 101;
private static final String DATAVERSE_NAME = "TestDV";
@@ -113,9 +116,9 @@
private static final IFieldValueGenerator[] SECONDARY_INDEX_VALUE_GENERATOR =
{ new AInt64ValueGenerator(), new AInt32ValueGenerator() };
private static final List<List<String>> INDEX_FIELD_NAMES =
- Arrays.asList(Arrays.asList(RECORD_TYPE.getFieldNames()[1]));
- private static final List<Integer> INDEX_FIELD_INDICATORS = Arrays.asList(Index.RECORD_INDICATOR);
- private static final List<IAType> INDEX_FIELD_TYPES = Arrays.asList(BuiltinType.AINT64);
+ Collections.singletonList(Collections.singletonList(RECORD_TYPE.getFieldNames()[1]));
+ private static final List<Integer> INDEX_FIELD_INDICATORS = Collections.singletonList(Index.RECORD_INDICATOR);
+ private static final List<IAType> INDEX_FIELD_TYPES = Collections.singletonList(BuiltinType.AINT64);
private static final StorageComponentProvider storageManager = new StorageComponentProvider();
private static TestNodeController nc;
private static NCAppRuntimeContext ncAppCtx;
@@ -154,6 +157,9 @@
TestHelper.deleteExistingInstanceFiles();
}
+ @Rule
+ public TestRule tracer = new TestMethodTracer();
+
@Before
public void createIndex() throws Exception {
List<List<String>> partitioningKeys = new ArrayList<>();
@@ -206,50 +212,47 @@
@Test
public void testCheckpointUpdatedWhenSecondaryIsEmpty() throws Exception {
- try {
- // create secondary
- createSecondaryIndex();
- actor.add(new Request(Request.Action.INSERT_PATCH));
- ensureDone(actor);
- // search now and ensure partition 0 has all the records
- StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
- // and that secondary index is empty
- Assert.assertTrue(secondaryLsmBtree.isCurrentMutableComponentEmpty());
- // flush
- actor.add(new Request(Request.Action.FLUSH_DATASET));
- ensureDone(actor);
- // ensure primary has a component
- Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+ // create secondary
+ createSecondaryIndex();
+ actor.add(new Request(Request.Action.INSERT_PATCH));
+ ensureDone(actor);
+ // search now and ensure partition 0 has all the records
+ StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+ // and that secondary index is empty
+ Assert.assertTrue(secondaryLsmBtree.isCurrentMutableComponentEmpty());
+ // flush
+ actor.add(new Request(Request.Action.FLUSH_DATASET));
+ ensureDone(actor);
+ // ensure primary has a component
+ Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
- // ensure secondary doesn't have a component
- Assert.assertEquals(0, secondaryLsmBtree.getDiskComponents().size());
- // ensure that current memory component index match
- Assert.assertEquals(secondaryLsmBtree.getCurrentMemoryComponentIndex(),
- primaryLsmBtree.getCurrentMemoryComponentIndex());
- // ensure both checkpoint files has the same component id as the last flushed component id
- ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
- LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
- long min = id.getMinId();
- // primary ref
- Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
- fileManagerField.setAccessible(true); //Make it accessible so you can access it
- ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
- final ResourceReference primaryRef = ResourceReference
- .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
- // secondary ref
- ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
- final ResourceReference secondaryRef = ResourceReference.of(
- secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
- IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
- Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
- Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
- } catch (Throwable e) {
- e.printStackTrace();
- throw e;
- }
+ // ensure secondary doesn't have a component
+ Assert.assertEquals(0, secondaryLsmBtree.getDiskComponents().size());
+ // ensure that current memory component index match
+ Assert.assertEquals(secondaryLsmBtree.getCurrentMemoryComponentIndex(),
+ primaryLsmBtree.getCurrentMemoryComponentIndex());
+ // ensure both checkpoint files has the same component id as the last flushed component id
+ ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+ LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+ long min = id.getMinId();
+ // primary ref
+ Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+ fileManagerField.setAccessible(true); //Make it accessible so you can access it
+ ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+ final ResourceReference primaryRef = ResourceReference
+ .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+ // secondary ref
+ ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+ final ResourceReference secondaryRef = ResourceReference
+ .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+ IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+ Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+ Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+ Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
}
private void createSecondaryIndex()
@@ -266,204 +269,192 @@
@Test
public void testCheckpointWhenBulkloadingSecondaryAndPrimaryIsSingleComponent() throws Exception {
- try {
- // create secondary
- actor.add(new Request(Request.Action.INSERT_PATCH));
- ensureDone(actor);
- // search now and ensure partition 0 has all the records
- StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
- // flush
- actor.add(new Request(Request.Action.FLUSH_DATASET));
- ensureDone(actor);
- // ensure primary has a component
- Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
- // ensure both checkpoint files has the same component id as the last flushed component id
- ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
- LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
- long min = id.getMinId();
- // primary ref
- Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
- fileManagerField.setAccessible(true); //Make it accessible so you can access it
- ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
- final ResourceReference primaryRef = ResourceReference
- .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
- IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
- createSecondaryIndex();
- JobId jobId = nc.newJobId();
- loadTaskCtx = nc.createTestContext(jobId, 0, false);
- Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
- nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
- KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
- indexLoadOp = infoAndOp.getRight();
- secondaryIndexInfo = infoAndOp.getLeft();
- actor.add(new Request(Request.Action.LOAD_OPEN));
- actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
- actor.add(new Request(Request.Action.LOAD_CLOSE));
- ensureDone(actor);
- latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
- ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
- final ResourceReference secondaryRef = ResourceReference.of(
- secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
- IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
- Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
- } catch (Throwable e) {
- e.printStackTrace();
- throw e;
- }
+ // create secondary
+ actor.add(new Request(Request.Action.INSERT_PATCH));
+ ensureDone(actor);
+ // search now and ensure partition 0 has all the records
+ StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+ // flush
+ actor.add(new Request(Request.Action.FLUSH_DATASET));
+ ensureDone(actor);
+ // ensure primary has a component
+ Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+ // ensure both checkpoint files has the same component id as the last flushed component id
+ ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+ LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+ long min = id.getMinId();
+ // primary ref
+ Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+ fileManagerField.setAccessible(true); //Make it accessible so you can access it
+ ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+ final ResourceReference primaryRef = ResourceReference
+ .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+ IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+ createSecondaryIndex();
+ JobId jobId = nc.newJobId();
+ loadTaskCtx = nc.createTestContext(jobId, 0, false);
+ Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+ nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+ KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+ indexLoadOp = infoAndOp.getRight();
+ secondaryIndexInfo = infoAndOp.getLeft();
+ actor.add(new Request(Request.Action.LOAD_OPEN));
+ actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
+ actor.add(new Request(Request.Action.LOAD_CLOSE));
+ ensureDone(actor);
+ latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+ ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+ final ResourceReference secondaryRef = ResourceReference
+ .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+ IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+ Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+ Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
}
@Test
public void testCheckpointWhenBulkloadingSecondaryAndPrimaryIsTwoComponents() throws Exception {
- try {
- // create secondary
- actor.add(new Request(Request.Action.INSERT_PATCH));
- ensureDone(actor);
- // search now and ensure partition 0 has all the records
- StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
- // flush
- actor.add(new Request(Request.Action.FLUSH_DATASET));
- ensureDone(actor);
- // ensure primary has a component
- Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
- // ensure both checkpoint files has the same component id as the last flushed component id
- ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
- LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
- long min = id.getMinId();
- // primary ref
- Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
- fileManagerField.setAccessible(true); //Make it accessible so you can access it
- ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
- final ResourceReference primaryRef = ResourceReference
- .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
- IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
- actor.add(new Request(Request.Action.INSERT_PATCH));
- ensureDone(actor);
- actor.add(new Request(Request.Action.FLUSH_DATASET));
- ensureDone(actor);
- Assert.assertEquals(2, primaryLsmBtree.getDiskComponents().size());
- // ensure both checkpoint files has the same component id as the last flushed component id
- primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
- id = (LSMComponentId) primaryDiskComponent.getId();
- min = id.getMaxId();
- createSecondaryIndex();
- JobId jobId = nc.newJobId();
- loadTaskCtx = nc.createTestContext(jobId, 0, false);
- Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
- nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
- KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
- indexLoadOp = infoAndOp.getRight();
- secondaryIndexInfo = infoAndOp.getLeft();
- actor.add(new Request(Request.Action.LOAD_OPEN));
- actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
- actor.add(new Request(Request.Action.LOAD_CLOSE));
- ensureDone(actor);
- latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
- ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
- final ResourceReference secondaryRef = ResourceReference.of(
- secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
- IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
- Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
- } catch (Throwable e) {
- e.printStackTrace();
- throw e;
- }
+ // create secondary
+ actor.add(new Request(Request.Action.INSERT_PATCH));
+ ensureDone(actor);
+ // search now and ensure partition 0 has all the records
+ StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+ // flush
+ actor.add(new Request(Request.Action.FLUSH_DATASET));
+ ensureDone(actor);
+ // ensure primary has a component
+ Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+ // ensure both checkpoint files has the same component id as the last flushed component id
+ ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+ LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+ long min = id.getMinId();
+ // primary ref
+ Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+ fileManagerField.setAccessible(true); //Make it accessible so you can access it
+ ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+ final ResourceReference primaryRef = ResourceReference
+ .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+ IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+ actor.add(new Request(Request.Action.INSERT_PATCH));
+ ensureDone(actor);
+ actor.add(new Request(Request.Action.FLUSH_DATASET));
+ ensureDone(actor);
+ Assert.assertEquals(2, primaryLsmBtree.getDiskComponents().size());
+ // ensure both checkpoint files has the same component id as the last flushed component id
+ primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+ id = (LSMComponentId) primaryDiskComponent.getId();
+ min = id.getMaxId();
+ createSecondaryIndex();
+ JobId jobId = nc.newJobId();
+ loadTaskCtx = nc.createTestContext(jobId, 0, false);
+ Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+ nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+ KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+ indexLoadOp = infoAndOp.getRight();
+ secondaryIndexInfo = infoAndOp.getLeft();
+ actor.add(new Request(Request.Action.LOAD_OPEN));
+ actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
+ actor.add(new Request(Request.Action.LOAD_CLOSE));
+ ensureDone(actor);
+ latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+ ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+ final ResourceReference secondaryRef = ResourceReference
+ .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+ IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+ Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+ Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
}
@Test
public void testCheckpointWhenBulkloadedSecondaryIsEmptyAndPrimaryIsEmpty() throws Exception {
- try {
- // ensure primary has no component
- Assert.assertEquals(0, primaryLsmBtree.getDiskComponents().size());
- // primary ref
- Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
- fileManagerField.setAccessible(true); //Make it accessible so you can access it
- ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
- final ResourceReference primaryRef = ResourceReference
- .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
- IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- createSecondaryIndex();
- JobId jobId = nc.newJobId();
- loadTaskCtx = nc.createTestContext(jobId, 0, false);
- Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
- nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
- KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
- indexLoadOp = infoAndOp.getRight();
- secondaryIndexInfo = infoAndOp.getLeft();
- actor.add(new Request(Request.Action.LOAD_OPEN));
- actor.add(new Request(Request.Action.LOAD_CLOSE));
- ensureDone(actor);
- latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
- final ResourceReference secondaryRef = ResourceReference.of(
- secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
- IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
- Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(),
- latestPrimaryCheckpoint.getLastComponentId());
- } catch (Throwable e) {
- e.printStackTrace();
- throw e;
- }
+ // ensure primary has no component
+ Assert.assertEquals(0, primaryLsmBtree.getDiskComponents().size());
+ // primary ref
+ Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+ fileManagerField.setAccessible(true); //Make it accessible so you can access it
+ ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+ final ResourceReference primaryRef = ResourceReference
+ .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+ IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ createSecondaryIndex();
+ JobId jobId = nc.newJobId();
+ loadTaskCtx = nc.createTestContext(jobId, 0, false);
+ Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+ nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+ KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+ indexLoadOp = infoAndOp.getRight();
+ secondaryIndexInfo = infoAndOp.getLeft();
+ actor.add(new Request(Request.Action.LOAD_OPEN));
+ actor.add(new Request(Request.Action.LOAD_CLOSE));
+ ensureDone(actor);
+ latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+ final ResourceReference secondaryRef = ResourceReference
+ .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+ IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(),
+ latestPrimaryCheckpoint.getLastComponentId());
+ Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+ Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
}
@Test
public void testCheckpointWhenBulkloadedSecondaryIsEmptyAndPrimaryIsNotEmpty() throws Exception {
- try {
- // create secondary
- actor.add(new Request(Request.Action.INSERT_PATCH));
- ensureDone(actor);
- // search now and ensure partition 0 has all the records
- StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
- // flush
- actor.add(new Request(Request.Action.FLUSH_DATASET));
- ensureDone(actor);
- // ensure primary has a component
- Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
- // ensure both checkpoint files has the same component id as the last flushed component id
- ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
- LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
- long min = id.getMinId();
- // primary ref
- Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
- fileManagerField.setAccessible(true); //Make it accessible so you can access it
- ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
- final ResourceReference primaryRef = ResourceReference
- .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
- IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
- createSecondaryIndex();
- JobId jobId = nc.newJobId();
- loadTaskCtx = nc.createTestContext(jobId, 0, false);
- Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
- nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
- KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
- indexLoadOp = infoAndOp.getRight();
- secondaryIndexInfo = infoAndOp.getLeft();
- actor.add(new Request(Request.Action.LOAD_OPEN));
- actor.add(new Request(Request.Action.LOAD_CLOSE));
- ensureDone(actor);
- latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
- Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
- ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
- final ResourceReference secondaryRef = ResourceReference.of(
- secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
- IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
- IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
- Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
- } catch (Throwable e) {
- e.printStackTrace();
- throw e;
- }
+ // create secondary
+ actor.add(new Request(Request.Action.INSERT_PATCH));
+ ensureDone(actor);
+ // search now and ensure partition 0 has all the records
+ StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+ // flush
+ actor.add(new Request(Request.Action.FLUSH_DATASET));
+ ensureDone(actor);
+ // ensure primary has a component
+ Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+ // ensure both checkpoint files has the same component id as the last flushed component id
+ ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+ LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+ long min = id.getMinId();
+ // primary ref
+ Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+ fileManagerField.setAccessible(true); //Make it accessible so you can access it
+ ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+ final ResourceReference primaryRef = ResourceReference
+ .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+ IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+ createSecondaryIndex();
+ JobId jobId = nc.newJobId();
+ loadTaskCtx = nc.createTestContext(jobId, 0, false);
+ Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+ nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+ KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+ indexLoadOp = infoAndOp.getRight();
+ secondaryIndexInfo = infoAndOp.getLeft();
+ actor.add(new Request(Request.Action.LOAD_OPEN));
+ actor.add(new Request(Request.Action.LOAD_CLOSE));
+ ensureDone(actor);
+ latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+ ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+ final ResourceReference secondaryRef = ResourceReference
+ .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+ IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+ IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+ Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+ Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+ Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
}
protected IIndexCheckpointManagerProvider getIndexCheckpointManagerProvider() {
@@ -566,9 +557,6 @@
default:
break;
}
- } catch (Throwable th) {
- th.printStackTrace();
- throw th;
} finally {
req.complete();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
index 5e99eec..0e62851 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
@@ -139,6 +139,16 @@
Set<String> getParticipantNodes();
/**
+ * @return the set of currently failed nodes
+ */
+ Set<String> getFailedNodes();
+
+ /**
+ * @return the set of cluster nodes
+ */
+ Set<String> getNodes();
+
+ /**
* Returns the IO devices configured for a Node Controller
*
* @param nodeId
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
index cb34600..3d0b9cb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
@@ -27,14 +27,18 @@
import org.apache.logging.log4j.Logger;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+@JsonIgnoreProperties(ignoreUnknown = true)
public class IndexCheckpoint {
private static final Logger LOGGER = LogManager.getLogger();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final long INITIAL_CHECKPOINT_ID = 0;
+ // TODO(mblow): remove this marker & related logic once we no longer are able to read indexes prior to the fix
+ private static final long HAS_NULL_MISSING_VALUES_FIX = -1;
private long id;
private long validComponentSequence;
private long lowWatermark;
@@ -48,6 +52,7 @@
firstCheckpoint.validComponentSequence = lastComponentSequence;
firstCheckpoint.lastComponentId = validComponentId;
firstCheckpoint.masterNodeFlushMap = new HashMap<>();
+ firstCheckpoint.masterNodeFlushMap.put(HAS_NULL_MISSING_VALUES_FIX, HAS_NULL_MISSING_VALUES_FIX);
return firstCheckpoint;
}
@@ -66,7 +71,7 @@
next.validComponentSequence = validComponentSequence;
next.masterNodeFlushMap = latest.getMasterNodeFlushMap();
// remove any lsn from the map that wont be used anymore
- next.masterNodeFlushMap.values().removeIf(lsn -> lsn <= lowWatermark);
+ next.masterNodeFlushMap.values().removeIf(lsn -> lsn <= lowWatermark && lsn != HAS_NULL_MISSING_VALUES_FIX);
return next;
}
@@ -94,6 +99,10 @@
return id;
}
+ public boolean hasNullMissingValuesFix() {
+ return masterNodeFlushMap.containsKey(HAS_NULL_MISSING_VALUES_FIX);
+ }
+
public String asJson() throws HyracksDataException {
try {
return OBJECT_MAPPER.writeValueAsString(this);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 8d3187b..16a479e 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -251,6 +251,18 @@
}
@Override
+ public synchronized Set<String> getFailedNodes() {
+ return new HashSet<>(failedNodes);
+ }
+
+ @Override
+ public synchronized Set<String> getNodes() {
+ Set<String> nodes = new HashSet<>(participantNodes);
+ nodes.addAll(failedNodes);
+ return nodes;
+ }
+
+ @Override
public synchronized Set<String> getParticipantNodes(boolean excludePendingRemoval) {
final Set<String> participantNodesCopy = getParticipantNodes();
if (excludePendingRemoval) {
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
index 60f5cb5..652abff 100644
--- a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
@@ -32,9 +32,9 @@
if "%1" == "" goto postopts
if "%1" == "-f" (
- set force=1
+ rem ignored, this is always the case
) else if "%1" == "-force" (
- set force=1
+ rem ignored, this is always the case
) else if "%1" == "-usage" (
goto :usage
) else if "%1" == "-help" (
@@ -80,23 +80,12 @@
"name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^
GET processid > %tempfile% 2> nul
-set severity=ERROR
-if "%force%" == "1" set severity=WARNING
-
for /F "skip=1" %%P in ('type %tempfile%') DO set found=1
if "%found%" == "1" (
- if "%force%" == "1" (
- echo %severity%: ${PRODUCT} processes are already running; -f[orce] specified, ignoring
- del %tempfile%
- ) else (
- echo %severity%: ${PRODUCT} processes are already running; aborting"
- echo.
- echo Re-run with -f to ignore, or run stop-sample-cluster.bat -f to forcibly terminate all running ${PRODUCT} processes:
- for /F "skip=1" %%P in ('type %tempfile%') DO @echo - %%P
- del %tempfile%
- exit /B 1
- )
+ echo WARNING: ${PRODUCT} processes are already running:
+ for /F "skip=1" %%P in ('type %tempfile%') DO @echo - %%P
+ del %tempfile%
)
goto :post_timestamp
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
index 34d156c..9268bed 100755
--- a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
@@ -20,14 +20,13 @@
function usage() {
echo
- echo Usage: $(basename $0) [-f[orce]]
+ echo Usage: $(basename $0)
echo
- echo " -f[orce] : Forces a start attempt when ${PRODUCT} processes are found to be running"
}
while [ -n "$1" ]; do
case $1 in
- -f|-force) force=1;;
+ -f|-force);; # ignored, this is always the case
-help|--help|-usage|--usage) usage; exit 0;;
*) echo "ERROR: unknown argument '$1'"; usage; exit 1;;
esac
@@ -115,22 +114,9 @@
&& echo "ERROR: sample cluster address (localhost:${LISTEN_PORT}) already in use" && exit 1
if ps -ef | grep 'java.*org\.apache\.hyracks\.control\.[cn]c\.\([CN]CDriver\|service\.NCService\)' > /tmp/$$_pids; then
- if [ $force ]; then
- severity=WARNING
- else
- severity=ERROR
- fi
- echo -n "${severity}: ${PRODUCT} processes are already running; "
- if [ $force ]; then
- echo "-f[orce] specified, ignoring"
- else
- echo "aborting"
- echo
- echo "Re-run with -f to ignore, or run stop-sample-cluster.sh -f to forcibly terminate all running ${PRODUCT} processes:"
- cat /tmp/$$_pids | sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
- rm /tmp/$$_pids
- exit 1
- fi
+ echo "WARNING: ${PRODUCT} processes are already running:"
+ cat /tmp/$$_pids | sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
+ rm /tmp/$$_pids
fi
rm /tmp/$$_pids
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
index 80647d4..97473ac 100755
--- a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
@@ -146,7 +146,7 @@
done
else
echo "re-run with -f|-force to forcibly terminate all ${PRODUCT} processes:"
- cat /tmp/pids | sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
+ cat /tmp/$$_pids | sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
fi
fi
rm /tmp/$$_pids
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
index e752cb3..a1a39cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
@@ -19,32 +19,27 @@
package org.apache.hyracks.api.util;
+import java.util.concurrent.TimeUnit;
+
public class StopWatch {
private long startTime = 0;
- private long stopTime = 0;
private long elapsedTime = 0;
public void start() {
elapsedTime = 0;
- startTime = System.currentTimeMillis();
+ startTime = System.nanoTime();
}
public void stop() {
- stopTime = System.currentTimeMillis();
- elapsedTime += stopTime - startTime;
+ elapsedTime += System.nanoTime() - startTime;
}
public void resume() {
- startTime = System.currentTimeMillis();
+ startTime = System.nanoTime();
}
- //elaspsed time in milliseconds
- public long getElapsedTime() {
- return elapsedTime;
+ public long elapsed(TimeUnit unit) {
+ return unit.convert(elapsedTime, TimeUnit.NANOSECONDS);
}
- //elaspsed time in seconds
- public long getElapsedTimeSecs() {
- return elapsedTime / 1000;
- }
}
\ No newline at end of file