Merge commit 'fab68d291e' from 'stabilization-f69489'

Change-Id: Idcabca9b2b126f891a12e60e4b427fdc4828f07f
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
index 8f40b3a..3da845d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
@@ -54,6 +54,7 @@
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.operators.LSMIndexBulkLoadOperatorNodePushable;
+import org.apache.asterix.test.base.TestMethodTracer;
 import org.apache.asterix.test.common.TestHelper;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -79,7 +80,9 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestRule;
 import org.junit.runners.Parameterized;
 
 public class CheckpointInSecondaryIndexTest {
@@ -101,7 +104,7 @@
     private static final boolean[] UNIQUE_META_FIELDS = null;
     private static final int[] KEY_INDEXES = { 0 };
     private static final int[] KEY_INDICATORS = { Index.RECORD_INDICATOR };
-    private static final List<Integer> KEY_INDICATORS_LIST = Arrays.asList(new Integer[] { Index.RECORD_INDICATOR });
+    private static final List<Integer> KEY_INDICATORS_LIST = Collections.singletonList(Index.RECORD_INDICATOR);
     private static final int RECORDS_PER_COMPONENT = 500;
     private static final int DATASET_ID = 101;
     private static final String DATAVERSE_NAME = "TestDV";
@@ -113,9 +116,9 @@
     private static final IFieldValueGenerator[] SECONDARY_INDEX_VALUE_GENERATOR =
             { new AInt64ValueGenerator(), new AInt32ValueGenerator() };
     private static final List<List<String>> INDEX_FIELD_NAMES =
-            Arrays.asList(Arrays.asList(RECORD_TYPE.getFieldNames()[1]));
-    private static final List<Integer> INDEX_FIELD_INDICATORS = Arrays.asList(Index.RECORD_INDICATOR);
-    private static final List<IAType> INDEX_FIELD_TYPES = Arrays.asList(BuiltinType.AINT64);
+            Collections.singletonList(Collections.singletonList(RECORD_TYPE.getFieldNames()[1]));
+    private static final List<Integer> INDEX_FIELD_INDICATORS = Collections.singletonList(Index.RECORD_INDICATOR);
+    private static final List<IAType> INDEX_FIELD_TYPES = Collections.singletonList(BuiltinType.AINT64);
     private static final StorageComponentProvider storageManager = new StorageComponentProvider();
     private static TestNodeController nc;
     private static NCAppRuntimeContext ncAppCtx;
@@ -154,6 +157,9 @@
         TestHelper.deleteExistingInstanceFiles();
     }
 
+    @Rule
+    public TestRule tracer = new TestMethodTracer();
+
     @Before
     public void createIndex() throws Exception {
         List<List<String>> partitioningKeys = new ArrayList<>();
@@ -206,50 +212,47 @@
 
     @Test
     public void testCheckpointUpdatedWhenSecondaryIsEmpty() throws Exception {
-        try {
-            // create secondary
-            createSecondaryIndex();
-            actor.add(new Request(Request.Action.INSERT_PATCH));
-            ensureDone(actor);
-            // search now and ensure partition 0 has all the records
-            StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
-            // and that secondary index is empty
-            Assert.assertTrue(secondaryLsmBtree.isCurrentMutableComponentEmpty());
-            // flush
-            actor.add(new Request(Request.Action.FLUSH_DATASET));
-            ensureDone(actor);
-            // ensure primary has a component
-            Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+        // create secondary
+        createSecondaryIndex();
+        actor.add(new Request(Request.Action.INSERT_PATCH));
+        ensureDone(actor);
+        // search now and ensure partition 0 has all the records
+        StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+        // and that secondary index is empty
+        Assert.assertTrue(secondaryLsmBtree.isCurrentMutableComponentEmpty());
+        // flush
+        actor.add(new Request(Request.Action.FLUSH_DATASET));
+        ensureDone(actor);
+        // ensure primary has a component
+        Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
 
-            // ensure secondary doesn't have a component
-            Assert.assertEquals(0, secondaryLsmBtree.getDiskComponents().size());
-            // ensure that current memory component index match
-            Assert.assertEquals(secondaryLsmBtree.getCurrentMemoryComponentIndex(),
-                    primaryLsmBtree.getCurrentMemoryComponentIndex());
-            // ensure both checkpoint files has the same component id as the last flushed component id
-            ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
-            LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
-            long min = id.getMinId();
-            // primary ref
-            Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
-            fileManagerField.setAccessible(true); //Make it accessible so you can access it
-            ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
-            final ResourceReference primaryRef = ResourceReference
-                    .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
-            // secondary ref
-            ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
-            final ResourceReference secondaryRef = ResourceReference.of(
-                    secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
-            IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
-            Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
-        } catch (Throwable e) {
-            e.printStackTrace();
-            throw e;
-        }
+        // ensure secondary doesn't have a component
+        Assert.assertEquals(0, secondaryLsmBtree.getDiskComponents().size());
+        // ensure that current memory component index match
+        Assert.assertEquals(secondaryLsmBtree.getCurrentMemoryComponentIndex(),
+                primaryLsmBtree.getCurrentMemoryComponentIndex());
+        // ensure both checkpoint files has the same component id as the last flushed component id
+        ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+        LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+        long min = id.getMinId();
+        // primary ref
+        Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+        fileManagerField.setAccessible(true); //Make it accessible so you can access it
+        ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+        final ResourceReference primaryRef = ResourceReference
+                .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+        // secondary ref
+        ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+        final ResourceReference secondaryRef = ResourceReference
+                .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+        IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+        Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+        Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+        Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
     }
 
     private void createSecondaryIndex()
@@ -266,204 +269,192 @@
 
     @Test
     public void testCheckpointWhenBulkloadingSecondaryAndPrimaryIsSingleComponent() throws Exception {
-        try {
-            // create secondary
-            actor.add(new Request(Request.Action.INSERT_PATCH));
-            ensureDone(actor);
-            // search now and ensure partition 0 has all the records
-            StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
-            // flush
-            actor.add(new Request(Request.Action.FLUSH_DATASET));
-            ensureDone(actor);
-            // ensure primary has a component
-            Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
-            // ensure both checkpoint files has the same component id as the last flushed component id
-            ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
-            LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
-            long min = id.getMinId();
-            // primary ref
-            Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
-            fileManagerField.setAccessible(true); //Make it accessible so you can access it
-            ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
-            final ResourceReference primaryRef = ResourceReference
-                    .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
-            IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
-            createSecondaryIndex();
-            JobId jobId = nc.newJobId();
-            loadTaskCtx = nc.createTestContext(jobId, 0, false);
-            Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
-                    nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
-                            KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
-            indexLoadOp = infoAndOp.getRight();
-            secondaryIndexInfo = infoAndOp.getLeft();
-            actor.add(new Request(Request.Action.LOAD_OPEN));
-            actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
-            actor.add(new Request(Request.Action.LOAD_CLOSE));
-            ensureDone(actor);
-            latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
-            ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
-            final ResourceReference secondaryRef = ResourceReference.of(
-                    secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
-            IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
-        } catch (Throwable e) {
-            e.printStackTrace();
-            throw e;
-        }
+        // create secondary
+        actor.add(new Request(Request.Action.INSERT_PATCH));
+        ensureDone(actor);
+        // search now and ensure partition 0 has all the records
+        StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+        // flush
+        actor.add(new Request(Request.Action.FLUSH_DATASET));
+        ensureDone(actor);
+        // ensure primary has a component
+        Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+        // ensure both checkpoint files has the same component id as the last flushed component id
+        ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+        LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+        long min = id.getMinId();
+        // primary ref
+        Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+        fileManagerField.setAccessible(true); //Make it accessible so you can access it
+        ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+        final ResourceReference primaryRef = ResourceReference
+                .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+        IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+        createSecondaryIndex();
+        JobId jobId = nc.newJobId();
+        loadTaskCtx = nc.createTestContext(jobId, 0, false);
+        Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+                nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+                        KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+        indexLoadOp = infoAndOp.getRight();
+        secondaryIndexInfo = infoAndOp.getLeft();
+        actor.add(new Request(Request.Action.LOAD_OPEN));
+        actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
+        actor.add(new Request(Request.Action.LOAD_CLOSE));
+        ensureDone(actor);
+        latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+        ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+        final ResourceReference secondaryRef = ResourceReference
+                .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+        IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+        Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+        Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
     }
 
     @Test
     public void testCheckpointWhenBulkloadingSecondaryAndPrimaryIsTwoComponents() throws Exception {
-        try {
-            // create secondary
-            actor.add(new Request(Request.Action.INSERT_PATCH));
-            ensureDone(actor);
-            // search now and ensure partition 0 has all the records
-            StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
-            // flush
-            actor.add(new Request(Request.Action.FLUSH_DATASET));
-            ensureDone(actor);
-            // ensure primary has a component
-            Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
-            // ensure both checkpoint files has the same component id as the last flushed component id
-            ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
-            LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
-            long min = id.getMinId();
-            // primary ref
-            Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
-            fileManagerField.setAccessible(true); //Make it accessible so you can access it
-            ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
-            final ResourceReference primaryRef = ResourceReference
-                    .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
-            IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
-            actor.add(new Request(Request.Action.INSERT_PATCH));
-            ensureDone(actor);
-            actor.add(new Request(Request.Action.FLUSH_DATASET));
-            ensureDone(actor);
-            Assert.assertEquals(2, primaryLsmBtree.getDiskComponents().size());
-            // ensure both checkpoint files has the same component id as the last flushed component id
-            primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
-            id = (LSMComponentId) primaryDiskComponent.getId();
-            min = id.getMaxId();
-            createSecondaryIndex();
-            JobId jobId = nc.newJobId();
-            loadTaskCtx = nc.createTestContext(jobId, 0, false);
-            Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
-                    nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
-                            KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
-            indexLoadOp = infoAndOp.getRight();
-            secondaryIndexInfo = infoAndOp.getLeft();
-            actor.add(new Request(Request.Action.LOAD_OPEN));
-            actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
-            actor.add(new Request(Request.Action.LOAD_CLOSE));
-            ensureDone(actor);
-            latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
-            ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
-            final ResourceReference secondaryRef = ResourceReference.of(
-                    secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
-            IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
-        } catch (Throwable e) {
-            e.printStackTrace();
-            throw e;
-        }
+        // create secondary
+        actor.add(new Request(Request.Action.INSERT_PATCH));
+        ensureDone(actor);
+        // search now and ensure partition 0 has all the records
+        StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+        // flush
+        actor.add(new Request(Request.Action.FLUSH_DATASET));
+        ensureDone(actor);
+        // ensure primary has a component
+        Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+        // ensure both checkpoint files has the same component id as the last flushed component id
+        ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+        LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+        long min = id.getMinId();
+        // primary ref
+        Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+        fileManagerField.setAccessible(true); //Make it accessible so you can access it
+        ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+        final ResourceReference primaryRef = ResourceReference
+                .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+        IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+        actor.add(new Request(Request.Action.INSERT_PATCH));
+        ensureDone(actor);
+        actor.add(new Request(Request.Action.FLUSH_DATASET));
+        ensureDone(actor);
+        Assert.assertEquals(2, primaryLsmBtree.getDiskComponents().size());
+        // ensure both checkpoint files has the same component id as the last flushed component id
+        primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+        id = (LSMComponentId) primaryDiskComponent.getId();
+        min = id.getMaxId();
+        createSecondaryIndex();
+        JobId jobId = nc.newJobId();
+        loadTaskCtx = nc.createTestContext(jobId, 0, false);
+        Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+                nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+                        KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+        indexLoadOp = infoAndOp.getRight();
+        secondaryIndexInfo = infoAndOp.getLeft();
+        actor.add(new Request(Request.Action.LOAD_OPEN));
+        actor.add(new Request(Request.Action.INDEX_LOAD_PATCH));
+        actor.add(new Request(Request.Action.LOAD_CLOSE));
+        ensureDone(actor);
+        latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+        ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+        final ResourceReference secondaryRef = ResourceReference
+                .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+        IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+        Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+        Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
     }
 
     @Test
     public void testCheckpointWhenBulkloadedSecondaryIsEmptyAndPrimaryIsEmpty() throws Exception {
-        try {
-            // ensure primary has no component
-            Assert.assertEquals(0, primaryLsmBtree.getDiskComponents().size());
-            // primary ref
-            Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
-            fileManagerField.setAccessible(true); //Make it accessible so you can access it
-            ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
-            final ResourceReference primaryRef = ResourceReference
-                    .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
-            IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            createSecondaryIndex();
-            JobId jobId = nc.newJobId();
-            loadTaskCtx = nc.createTestContext(jobId, 0, false);
-            Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
-                    nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
-                            KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
-            indexLoadOp = infoAndOp.getRight();
-            secondaryIndexInfo = infoAndOp.getLeft();
-            actor.add(new Request(Request.Action.LOAD_OPEN));
-            actor.add(new Request(Request.Action.LOAD_CLOSE));
-            ensureDone(actor);
-            latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
-            final ResourceReference secondaryRef = ResourceReference.of(
-                    secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
-            IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(),
-                    latestPrimaryCheckpoint.getLastComponentId());
-        } catch (Throwable e) {
-            e.printStackTrace();
-            throw e;
-        }
+        // ensure primary has no component
+        Assert.assertEquals(0, primaryLsmBtree.getDiskComponents().size());
+        // primary ref
+        Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+        fileManagerField.setAccessible(true); //Make it accessible so you can access it
+        ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+        final ResourceReference primaryRef = ResourceReference
+                .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+        IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        createSecondaryIndex();
+        JobId jobId = nc.newJobId();
+        loadTaskCtx = nc.createTestContext(jobId, 0, false);
+        Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+                nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+                        KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+        indexLoadOp = infoAndOp.getRight();
+        secondaryIndexInfo = infoAndOp.getLeft();
+        actor.add(new Request(Request.Action.LOAD_OPEN));
+        actor.add(new Request(Request.Action.LOAD_CLOSE));
+        ensureDone(actor);
+        latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+        final ResourceReference secondaryRef = ResourceReference
+                .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+        IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(),
+                latestPrimaryCheckpoint.getLastComponentId());
+        Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+        Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
     }
 
     @Test
     public void testCheckpointWhenBulkloadedSecondaryIsEmptyAndPrimaryIsNotEmpty() throws Exception {
-        try {
-            // create secondary
-            actor.add(new Request(Request.Action.INSERT_PATCH));
-            ensureDone(actor);
-            // search now and ensure partition 0 has all the records
-            StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
-            // flush
-            actor.add(new Request(Request.Action.FLUSH_DATASET));
-            ensureDone(actor);
-            // ensure primary has a component
-            Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
-            // ensure both checkpoint files has the same component id as the last flushed component id
-            ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
-            LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
-            long min = id.getMinId();
-            // primary ref
-            Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
-            fileManagerField.setAccessible(true); //Make it accessible so you can access it
-            ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
-            final ResourceReference primaryRef = ResourceReference
-                    .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
-            IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
-            createSecondaryIndex();
-            JobId jobId = nc.newJobId();
-            loadTaskCtx = nc.createTestContext(jobId, 0, false);
-            Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
-                    nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
-                            KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
-            indexLoadOp = infoAndOp.getRight();
-            secondaryIndexInfo = infoAndOp.getLeft();
-            actor.add(new Request(Request.Action.LOAD_OPEN));
-            actor.add(new Request(Request.Action.LOAD_CLOSE));
-            ensureDone(actor);
-            latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
-            ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
-            final ResourceReference secondaryRef = ResourceReference.of(
-                    secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
-            IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
-            IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
-            Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
-        } catch (Throwable e) {
-            e.printStackTrace();
-            throw e;
-        }
+        // create secondary
+        actor.add(new Request(Request.Action.INSERT_PATCH));
+        ensureDone(actor);
+        // search now and ensure partition 0 has all the records
+        StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, RECORDS_PER_COMPONENT);
+        // flush
+        actor.add(new Request(Request.Action.FLUSH_DATASET));
+        ensureDone(actor);
+        // ensure primary has a component
+        Assert.assertEquals(1, primaryLsmBtree.getDiskComponents().size());
+        // ensure both checkpoint files has the same component id as the last flushed component id
+        ILSMDiskComponent primaryDiskComponent = primaryLsmBtree.getDiskComponents().get(0);
+        LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId();
+        long min = id.getMinId();
+        // primary ref
+        Field fileManagerField = AbstractLSMIndex.class.getDeclaredField("fileManager"); //get private declared object from class
+        fileManagerField.setAccessible(true); //Make it accessible so you can access it
+        ILSMIndexFileManager primaryFileManager = (ILSMIndexFileManager) fileManagerField.get(primaryLsmBtree);
+        final ResourceReference primaryRef = ResourceReference
+                .of(primaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager primaryCheckpointManager = getIndexCheckpointManagerProvider().get(primaryRef);
+        IndexCheckpoint latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+        createSecondaryIndex();
+        JobId jobId = nc.newJobId();
+        loadTaskCtx = nc.createTestContext(jobId, 0, false);
+        Pair<SecondaryIndexInfo, LSMIndexBulkLoadOperatorNodePushable> infoAndOp =
+                nc.getBulkLoadSecondaryOperator(loadTaskCtx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null,
+                        KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, secondaryIndex, RECORDS_PER_COMPONENT);
+        indexLoadOp = infoAndOp.getRight();
+        secondaryIndexInfo = infoAndOp.getLeft();
+        actor.add(new Request(Request.Action.LOAD_OPEN));
+        actor.add(new Request(Request.Action.LOAD_CLOSE));
+        ensureDone(actor);
+        latestPrimaryCheckpoint = primaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestPrimaryCheckpoint.getLastComponentId(), min);
+        ILSMIndexFileManager secondaryFileManager = (ILSMIndexFileManager) fileManagerField.get(secondaryLsmBtree);
+        final ResourceReference secondaryRef = ResourceReference
+                .of(secondaryFileManager.getRelFlushFileReference().getInsertIndexFileReference().getAbsolutePath());
+        IIndexCheckpointManager secondaryCheckpointManager = getIndexCheckpointManagerProvider().get(secondaryRef);
+        IndexCheckpoint latestSecondaryCheckpoint = secondaryCheckpointManager.getLatest();
+        Assert.assertEquals(latestSecondaryCheckpoint.getLastComponentId(), min);
+        Assert.assertTrue(latestPrimaryCheckpoint.hasNullMissingValuesFix());
+        Assert.assertTrue(latestSecondaryCheckpoint.hasNullMissingValuesFix());
     }
 
     protected IIndexCheckpointManagerProvider getIndexCheckpointManagerProvider() {
@@ -566,9 +557,6 @@
                     default:
                         break;
                 }
-            } catch (Throwable th) {
-                th.printStackTrace();
-                throw th;
             } finally {
                 req.complete();
             }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
index 5e99eec..0e62851 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
@@ -139,6 +139,16 @@
     Set<String> getParticipantNodes();
 
     /**
+     * @return the set of currently failed nodes
+     */
+    Set<String> getFailedNodes();
+
+    /**
+     * @return the set of cluster nodes
+     */
+    Set<String> getNodes();
+
+    /**
      * Returns the IO devices configured for a Node Controller
      *
      * @param nodeId
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
index cb34600..3d0b9cb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
@@ -27,14 +27,18 @@
 import org.apache.logging.log4j.Logger;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class IndexCheckpoint {
 
     private static final Logger LOGGER = LogManager.getLogger();
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
     private static final long INITIAL_CHECKPOINT_ID = 0;
+    // TODO(mblow): remove this marker & related logic once we no longer are able to read indexes prior to the fix
+    private static final long HAS_NULL_MISSING_VALUES_FIX = -1;
     private long id;
     private long validComponentSequence;
     private long lowWatermark;
@@ -48,6 +52,7 @@
         firstCheckpoint.validComponentSequence = lastComponentSequence;
         firstCheckpoint.lastComponentId = validComponentId;
         firstCheckpoint.masterNodeFlushMap = new HashMap<>();
+        firstCheckpoint.masterNodeFlushMap.put(HAS_NULL_MISSING_VALUES_FIX, HAS_NULL_MISSING_VALUES_FIX);
         return firstCheckpoint;
     }
 
@@ -66,7 +71,7 @@
         next.validComponentSequence = validComponentSequence;
         next.masterNodeFlushMap = latest.getMasterNodeFlushMap();
         // remove any lsn from the map that wont be used anymore
-        next.masterNodeFlushMap.values().removeIf(lsn -> lsn <= lowWatermark);
+        next.masterNodeFlushMap.values().removeIf(lsn -> lsn <= lowWatermark && lsn != HAS_NULL_MISSING_VALUES_FIX);
         return next;
     }
 
@@ -94,6 +99,10 @@
         return id;
     }
 
+    public boolean hasNullMissingValuesFix() {
+        return masterNodeFlushMap.containsKey(HAS_NULL_MISSING_VALUES_FIX);
+    }
+
     public String asJson() throws HyracksDataException {
         try {
             return OBJECT_MAPPER.writeValueAsString(this);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 8d3187b..16a479e 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -251,6 +251,18 @@
     }
 
     @Override
+    public synchronized Set<String> getFailedNodes() {
+        return new HashSet<>(failedNodes);
+    }
+
+    @Override
+    public synchronized Set<String> getNodes() {
+        Set<String> nodes = new HashSet<>(participantNodes);
+        nodes.addAll(failedNodes);
+        return nodes;
+    }
+
+    @Override
     public synchronized Set<String> getParticipantNodes(boolean excludePendingRemoval) {
         final Set<String> participantNodesCopy = getParticipantNodes();
         if (excludePendingRemoval) {
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
index 60f5cb5..652abff 100644
--- a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
@@ -32,9 +32,9 @@
 if "%1" == "" goto postopts
 
 if "%1" == "-f" (
-  set force=1
+  rem ignored, this is always the case
 ) else if "%1" == "-force" (
-  set force=1
+  rem ignored, this is always the case
 ) else if "%1" == "-usage" (
   goto :usage
 ) else if "%1" == "-help" (
@@ -80,23 +80,12 @@
   "name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^
   GET processid > %tempfile% 2> nul
 
-set severity=ERROR
-if "%force%" == "1" set severity=WARNING
-
 for /F "skip=1" %%P in ('type %tempfile%') DO set found=1
 
 if "%found%" == "1" (
-  if "%force%" == "1" (
-    echo %severity%: ${PRODUCT} processes are already running; -f[orce] specified, ignoring
-    del %tempfile%
- ) else (
-    echo %severity%: ${PRODUCT} processes are already running; aborting"
-    echo.
-    echo Re-run with -f to ignore, or run stop-sample-cluster.bat -f to forcibly terminate all running ${PRODUCT} processes:
-    for /F "skip=1" %%P in ('type %tempfile%') DO @echo     - %%P
-    del %tempfile%
-    exit /B 1
-  )
+  echo WARNING: ${PRODUCT} processes are already running:
+  for /F "skip=1" %%P in ('type %tempfile%') DO @echo     - %%P
+  del %tempfile%
 )
 
 goto :post_timestamp
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
index 34d156c..9268bed 100755
--- a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
@@ -20,14 +20,13 @@
 
 function usage() {
   echo
-  echo Usage: $(basename $0) [-f[orce]]
+  echo Usage: $(basename $0)
   echo
-  echo "  -f[orce]  : Forces a start attempt when ${PRODUCT} processes are found to be running"
 }
 
 while [ -n "$1" ]; do
   case $1 in
-    -f|-force) force=1;;
+    -f|-force);; # ignored, this is always the case
     -help|--help|-usage|--usage) usage; exit 0;;
     *) echo "ERROR: unknown argument '$1'"; usage; exit 1;;
   esac
@@ -115,22 +114,9 @@
     && echo "ERROR: sample cluster address (localhost:${LISTEN_PORT}) already in use" && exit 1
 
 if ps -ef | grep 'java.*org\.apache\.hyracks\.control\.[cn]c\.\([CN]CDriver\|service\.NCService\)' > /tmp/$$_pids; then
-  if [ $force ]; then
-    severity=WARNING
-  else
-    severity=ERROR
-  fi
-  echo -n "${severity}: ${PRODUCT} processes are already running; "
-  if [ $force ]; then
-    echo "-f[orce] specified, ignoring"
-  else
-    echo "aborting"
-    echo
-    echo "Re-run with -f to ignore, or run stop-sample-cluster.sh -f to forcibly terminate all running ${PRODUCT} processes:"
-    cat /tmp/$$_pids |  sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
-    rm /tmp/$$_pids
-    exit 1
-  fi
+  echo "WARNING: ${PRODUCT} processes are already running:"
+  cat /tmp/$$_pids |  sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
+  rm /tmp/$$_pids
 fi
 
 rm /tmp/$$_pids
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
index 80647d4..97473ac 100755
--- a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
@@ -146,7 +146,7 @@
     done
   else
     echo "re-run with -f|-force to forcibly terminate all ${PRODUCT} processes:"
-    cat /tmp/pids |  sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
+    cat /tmp/$$_pids |  sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
   fi
 fi
 rm /tmp/$$_pids
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
index e752cb3..a1a39cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
@@ -19,32 +19,27 @@
 
 package org.apache.hyracks.api.util;
 
+import java.util.concurrent.TimeUnit;
+
 public class StopWatch {
     private long startTime = 0;
-    private long stopTime = 0;
     private long elapsedTime = 0;
 
     public void start() {
         elapsedTime = 0;
-        startTime = System.currentTimeMillis();
+        startTime = System.nanoTime();
     }
 
     public void stop() {
-        stopTime = System.currentTimeMillis();
-        elapsedTime += stopTime - startTime;
+        elapsedTime += System.nanoTime() - startTime;
     }
 
     public void resume() {
-        startTime = System.currentTimeMillis();
+        startTime = System.nanoTime();
     }
 
-    //elaspsed time in milliseconds
-    public long getElapsedTime() {
-        return elapsedTime;
+    public long elapsed(TimeUnit unit) {
+        return unit.convert(elapsedTime, TimeUnit.NANOSECONDS);
     }
 
-    //elaspsed time in seconds
-    public long getElapsedTimeSecs() {
-        return elapsedTime / 1000;
-    }
 }
\ No newline at end of file