diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
index bee4906..e15ffe0 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
@@ -45,5 +45,6 @@
     INSERT_DELETE,
     INDEX_INSERT_DELETE,
     UPDATE,
-    EXTENSION_OPERATOR
+    EXTENSION_OPERATOR,
+    EXTERNAL_LOOKUP
 }
\ No newline at end of file
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
index f6de971..c41107b 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
@@ -61,5 +61,6 @@
     SINGLE_PARTITION_INVERTED_INDEX_SEARCH,
     LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH,
     PARTITIONINGSPLIT,
-    EXTENSION_OPERATOR
+    EXTENSION_OPERATOR,
+    EXTERNAL_LOOKUP
 }
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
new file mode 100644
index 0000000..e18c461
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ExternalDataLookupOperator extends AbstractScanOperator {
+
+    private final List<Object> variableTypes;
+    protected final Mutable<ILogicalExpression> expression;
+    private final boolean propagateInput;
+
+    public ExternalDataLookupOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
+            List<Object> variableTypes, boolean propagateInput) {
+        super(variables);
+        this.expression = expression;
+        this.variableTypes = variableTypes;
+        this.propagateInput = propagateInput;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.EXTERNAL_LOOKUP;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = null;
+        if (propagateInput) {
+            env = createPropagatingAllInputsTypeEnvironment(ctx);
+        } else {
+            env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getMetadataProvider());
+        }
+        env.setVarType(variables.get(0), variableTypes.get(0));
+        return env;
+    }
+
+    public List<Object> getVariableTypes() {
+        return variableTypes;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitExternalDataLookupOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    public boolean isPropagateInput() {
+        return propagateInput;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                if (propagateInput) {
+                    ArrayList<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+                    VariableUtilities.getUsedVariables(ExternalDataLookupOperator.this, usedVariables);
+                    int numOfSources = sources.length;
+                    for (int i = 0; i < numOfSources; i++) {
+                        for (LogicalVariable v : sources[i]) {
+                            if (!usedVariables.contains(v)) {
+                                target.addVariable(v);
+                            }
+                        }
+                    }
+                }
+                target.addVariable(variables.get(0));
+            }
+        };
+    }
+
+    public Mutable<ILogicalExpression> getExpressionRef() {
+        return expression;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return visitor.transform(expression);
+    }
+}
\ No newline at end of file
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
index 3a2617f..ae0346e 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
@@ -50,6 +50,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -647,4 +648,11 @@
         return null;
     }
 
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
 }
\ No newline at end of file
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index 9157878..9e94a5b 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -41,6 +41,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -666,7 +667,8 @@
 
         @Override
         public ILogicalOperator visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
-            return new SelectOperator(deepCopyExpressionRef(op.getCondition()), op.getRetainNull(), op.getNullPlaceholderVariable());
+            return new SelectOperator(deepCopyExpressionRef(op.getCondition()), op.getRetainNull(),
+                    op.getNullPlaceholderVariable());
         }
 
         @Override
@@ -824,6 +826,21 @@
         public ILogicalOperator visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
             return new ExtensionOperator(op.getNewInstanceOfDelegateOperator());
         }
+
+        @Override
+        public ILogicalOperator visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
+                throws AlgebricksException {
+            ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+            newInputList.addAll(op.getVariables());
+            return new ExternalDataLookupOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()),
+                    new ArrayList<Object>(op.getVariableTypes()), op.isPropagateInput());
+        }
+    }
+
+    @Override
+    public Boolean visitExternalDataLookupOperator(ExternalDataLookupOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        return Boolean.FALSE;
     }
 
 }
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
index eecace7..ef91b38 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
@@ -40,6 +40,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -422,4 +423,10 @@
         return null;
     }
 
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        return null;
+    }
+
 }
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
index 277b5a3..225edd6 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
@@ -32,6 +32,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -325,4 +326,10 @@
         return null;
     }
 
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        return null;
+    }
+
 }
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
index 8673a77..3b8ed56 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
@@ -12,244 +12,251 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class ProducedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
-
-    private Collection<LogicalVariable> producedVariables;
-
-    public ProducedVariableVisitor(Collection<LogicalVariable> producedVariables) throws AlgebricksException {
-        this.producedVariables = producedVariables;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
-            if (p.first != null) {
-                producedVariables.add(p.first);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
-            if (p.first != null) {
-                producedVariables.add(p.first);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
-        List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op, usedVariables);
-        for (LogicalVariable v : op.getOutputVariables()) {
-            if (!usedVariables.contains(v)) {
-                producedVariables.add(v);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : op.getVariableMappings()) {
-            producedVariables.add(t.third);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        LogicalVariable positionalVariable = op.getPositionalVariable();
-        if (positionalVariable != null) {
-            if (!producedVariables.contains(positionalVariable))
-                producedVariables.add(positionalVariable);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
-        op.getDelegate().getProducedVariables(producedVariables);
-        return null;
-    }
-}
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ProducedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+    private Collection<LogicalVariable> producedVariables;
+
+    public ProducedVariableVisitor(Collection<LogicalVariable> producedVariables) throws AlgebricksException {
+        this.producedVariables = producedVariables;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
+            if (p.first != null) {
+                producedVariables.add(p.first);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
+            if (p.first != null) {
+                producedVariables.add(p.first);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+        List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op, usedVariables);
+        for (LogicalVariable v : op.getOutputVariables()) {
+            if (!usedVariables.contains(v)) {
+                producedVariables.add(v);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : op.getVariableMappings()) {
+            producedVariables.add(t.third);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        LogicalVariable positionalVariable = op.getPositionalVariable();
+        if (positionalVariable != null) {
+            if (!producedVariables.contains(positionalVariable))
+                producedVariables.add(positionalVariable);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+        op.getDelegate().getProducedVariables(producedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
+        producedVariables.add(op.getVariables().get(0));
+        return null;
+    }
+}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
index 5606e72..1d0fd7c 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
@@ -12,280 +12,301 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class SchemaVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
-
-    private Collection<LogicalVariable> schemaVariables;
-
-    public SchemaVariableVisitor(Collection<LogicalVariable> schemaVariables) {
-        this.schemaVariables = schemaVariables;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
-        schemaVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
-        List<LogicalVariable> allLiveVars = new ArrayList<LogicalVariable>();
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getValue(), allLiveVars);
-        }
-        VariableUtilities.getProducedVariables(op, allLiveVars);
-        /** put distinct vars first */
-        schemaVariables.addAll(op.getDistinctByVarList());
-        /** then other live vars */
-        for (LogicalVariable var : allLiveVars) {
-            if (!schemaVariables.contains(var)) {
-                schemaVariables.add(var);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
-            if (p.first != null) {
-                schemaVariables.add(p.first);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
-            if (p.first != null) {
-                schemaVariables.add(p.first);
-            } else {
-                ILogicalExpression e = p.second.getValue();
-                if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                    schemaVariables.add(((VariableReferenceExpression) e).getVariableReference());
-                }
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
-        VariableUtilities.getLiveVariables(op.getSourceOperator(), schemaVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
-        schemaVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
-        // VariableUtilities.getProducedVariables(op, schemaVariables);
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
-        schemaVariables.addAll(op.getOutputVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
-        }
-        VariableUtilities.getProducedVariables(op, schemaVariables);
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
-        VariableUtilities.getProducedVariables(op, schemaVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    private void standardLayout(ILogicalOperator op) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
-        }
-        VariableUtilities.getProducedVariables(op, schemaVariables);
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-}
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class SchemaVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+    private Collection<LogicalVariable> schemaVariables;
+
+    public SchemaVariableVisitor(Collection<LogicalVariable> schemaVariables) {
+        this.schemaVariables = schemaVariables;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+        schemaVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
+        List<LogicalVariable> allLiveVars = new ArrayList<LogicalVariable>();
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), allLiveVars);
+        }
+        VariableUtilities.getProducedVariables(op, allLiveVars);
+        /** put distinct vars first */
+        schemaVariables.addAll(op.getDistinctByVarList());
+        /** then other live vars */
+        for (LogicalVariable var : allLiveVars) {
+            if (!schemaVariables.contains(var)) {
+                schemaVariables.add(var);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
+            if (p.first != null) {
+                schemaVariables.add(p.first);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
+            if (p.first != null) {
+                schemaVariables.add(p.first);
+            } else {
+                ILogicalExpression e = p.second.getValue();
+                if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                    schemaVariables.add(((VariableReferenceExpression) e).getVariableReference());
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
+        VariableUtilities.getLiveVariables(op.getSourceOperator(), schemaVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+        schemaVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
+        // VariableUtilities.getProducedVariables(op, schemaVariables);
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+        schemaVariables.addAll(op.getOutputVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
+        }
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    private void standardLayout(ILogicalOperator op) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
+        }
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> liveVariables = new ArrayList<LogicalVariable>();
+        ArrayList<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+        //get used variables
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        //live variables - used variables
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), liveVariables);
+        }
+        for (LogicalVariable v : liveVariables) {
+            if (!usedVariables.contains(v)) {
+                schemaVariables.add(v);
+            }
+        }
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+        //+ produced variables
+        return null;
+    }
+
+}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
index 3d6b989..729a996 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -12,410 +12,426 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class SubstituteVariableVisitor implements ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> {
-
-    private final boolean goThroughNts;
-    private final ITypingContext ctx;
-
-    public SubstituteVariableVisitor(boolean goThroughNts, ITypingContext ctx) {
-        this.goThroughNts = goThroughNts;
-        this.ctx = ctx;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            if (variables.get(i).equals(pair.first)) {
-                variables.set(i, pair.second);
-            } else {
-                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            if (variables.get(i).equals(pair.first)) {
-                variables.set(i, pair.second);
-            } else {
-                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        for (int i = 0; i < variables.size(); i++) {
-            if (variables.get(i) == pair.first) {
-                variables.set(i, pair.second);
-                return null;
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
-            eRef.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        subst(pair.first, pair.second, op.getGroupByList());
-        subst(pair.first, pair.second, op.getDecorList());
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
-                        pair.second, goThroughNts, ctx);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getCondition().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getCondition().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getMaxObjects().getValue().substituteVar(pair.first, pair.second);
-        ILogicalExpression offset = op.getOffset().getValue();
-        if (offset != null) {
-            offset.substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
-            oe.second.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> usedVariables = op.getVariables();
-        int n = usedVariables.size();
-        for (int i = 0; i < n; i++) {
-            LogicalVariable v = usedVariables.get(i);
-            if (v.equals(pair.first)) {
-                usedVariables.set(i, pair.second);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            if (variables.get(i).equals(pair.first)) {
-                variables.set(i, pair.second);
-            } else {
-                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        substInArray(op.getInputVariables(), pair.first, pair.second);
-        substInArray(op.getOutputVariables(), pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
-        op.getCondition().getValue().substituteVar(pair.first, pair.second);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
-                        pair.second, goThroughNts, ctx);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
-            if (t.first.equals(pair.first)) {
-                t.first = pair.second;
-            }
-            if (t.second.equals(pair.first)) {
-                t.second = pair.second;
-            }
-            if (t.third.equals(pair.first)) {
-                t.third = pair.second;
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        for (int i = 0; i < variables.size(); i++) {
-            if (variables.get(i) == pair.first) {
-                variables.set(i, pair.second);
-                return null;
-            }
-        }
-        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        for (int i = 0; i < variables.size(); i++) {
-            if (variables.get(i) == pair.first) {
-                variables.set(i, pair.second);
-                return null;
-            }
-        }
-        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
-        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    private void subst(LogicalVariable v1, LogicalVariable v2,
-            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> varExprPairList) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : varExprPairList) {
-            if (ve.first != null && ve.first.equals(v1)) {
-                ve.first = v2;
-                return;
-            }
-            ve.second.getValue().substituteVar(v1, v2);
-        }
-    }
-
-    private void substInArray(ArrayList<LogicalVariable> varArray, LogicalVariable v1, LogicalVariable v2) {
-        for (int i = 0; i < varArray.size(); i++) {
-            LogicalVariable v = varArray.get(i);
-            if (v == v1) {
-                varArray.set(i, v2);
-            }
-        }
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Pair<LogicalVariable, LogicalVariable> arg)
-            throws AlgebricksException {
-        op.substituteVar(arg.first, arg.second);
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        return null;
-    }
-
-    private void substVarTypes(ILogicalOperator op, Pair<LogicalVariable, LogicalVariable> arg)
-            throws AlgebricksException {
-        if (ctx == null) {
-            return;
-        }
-        IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(op);
-        env.substituteProducedVariable(arg.first, arg.second);
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Pair<LogicalVariable, LogicalVariable> arg)
-            throws AlgebricksException {
-        return null;
-    }
-}
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class SubstituteVariableVisitor implements ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> {
+
+    private final boolean goThroughNts;
+    private final ITypingContext ctx;
+
+    public SubstituteVariableVisitor(boolean goThroughNts, ITypingContext ctx) {
+        this.goThroughNts = goThroughNts;
+        this.ctx = ctx;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            if (variables.get(i).equals(pair.first)) {
+                variables.set(i, pair.second);
+            } else {
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            if (variables.get(i).equals(pair.first)) {
+                variables.set(i, pair.second);
+            } else {
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+            eRef.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        subst(pair.first, pair.second, op.getGroupByList());
+        subst(pair.first, pair.second, op.getDecorList());
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
+                        pair.second, goThroughNts, ctx);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getMaxObjects().getValue().substituteVar(pair.first, pair.second);
+        ILogicalExpression offset = op.getOffset().getValue();
+        if (offset != null) {
+            offset.substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+            oe.second.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> usedVariables = op.getVariables();
+        int n = usedVariables.size();
+        for (int i = 0; i < n; i++) {
+            LogicalVariable v = usedVariables.get(i);
+            if (v.equals(pair.first)) {
+                usedVariables.set(i, pair.second);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            if (variables.get(i).equals(pair.first)) {
+                variables.set(i, pair.second);
+            } else {
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        substInArray(op.getInputVariables(), pair.first, pair.second);
+        substInArray(op.getOutputVariables(), pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
+                        pair.second, goThroughNts, ctx);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+            if (t.first.equals(pair.first)) {
+                t.first = pair.second;
+            }
+            if (t.second.equals(pair.first)) {
+                t.second = pair.second;
+            }
+            if (t.third.equals(pair.first)) {
+                t.third = pair.second;
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    private void subst(LogicalVariable v1, LogicalVariable v2,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> varExprPairList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : varExprPairList) {
+            if (ve.first != null && ve.first.equals(v1)) {
+                ve.first = v2;
+                return;
+            }
+            ve.second.getValue().substituteVar(v1, v2);
+        }
+    }
+
+    private void substInArray(ArrayList<LogicalVariable> varArray, LogicalVariable v1, LogicalVariable v2) {
+        for (int i = 0; i < varArray.size(); i++) {
+            LogicalVariable v = varArray.get(i);
+            if (v == v1) {
+                varArray.set(i, v2);
+            }
+        }
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+            throws AlgebricksException {
+        op.substituteVar(arg.first, arg.second);
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        return null;
+    }
+
+    private void substVarTypes(ILogicalOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+            throws AlgebricksException {
+        if (ctx == null) {
+            return;
+        }
+        IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(op);
+        env.substituteProducedVariable(arg.first, arg.second);
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+            throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op,
+            Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
index d05adfb..8dc545f 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -12,339 +12,346 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class UsedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
-
-    private Collection<LogicalVariable> usedVariables;
-
-    public UsedVariableVisitor(Collection<LogicalVariable> usedVariables) {
-        this.usedVariables = usedVariables;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
-            exprRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
-            exprRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
-            eRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
-        // Used variables depend on the physical operator.
-        if (op.getPhysicalOperator() != null) {
-            IPhysicalOperator physOp = op.getPhysicalOperator();
-            switch (physOp.getOperatorTag()) {
-                case BROADCAST_EXCHANGE:
-                case ONE_TO_ONE_EXCHANGE:
-                case RANDOM_MERGE_EXCHANGE: {
-                    // No variables used.
-                    break;
-                }
-                case HASH_PARTITION_EXCHANGE: {
-                    HashPartitionExchangePOperator concreteOp = (HashPartitionExchangePOperator) physOp;
-                    usedVariables.addAll(concreteOp.getHashFields());
-                    break;
-                }
-                case HASH_PARTITION_MERGE_EXCHANGE: {
-                    HashPartitionMergeExchangePOperator concreteOp = (HashPartitionMergeExchangePOperator) physOp;
-                    usedVariables.addAll(concreteOp.getPartitionFields());
-                    for (OrderColumn orderCol : concreteOp.getOrderColumns()) {
-                        usedVariables.add(orderCol.getColumn());
-                    }
-                    break;
-                }
-                case SORT_MERGE_EXCHANGE: {
-                    SortMergeExchangePOperator concreteOp = (SortMergeExchangePOperator) physOp;
-                    for (OrderColumn orderCol : concreteOp.getSortColumns()) {
-                        usedVariables.add(orderCol.getColumn());
-                    }
-                    break;
-                }
-                case RANGE_PARTITION_EXCHANGE: {
-                    RangePartitionPOperator concreteOp = (RangePartitionPOperator) physOp;
-                    for (OrderColumn partCol : concreteOp.getPartitioningFields()) {
-                        usedVariables.add(partCol.getColumn());
-                    }
-                    break;
-                }
-                default: {
-                    throw new AlgebricksException("Unhandled physical operator tag '" + physOp.getOperatorTag() + "'.");
-                }
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getGroupByList()) {
-            g.second.getValue().getUsedVariables(usedVariables);
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getDecorList()) {
-            g.second.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) {
-        op.getCondition().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) {
-        op.getCondition().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Void arg) {
-        op.getMaxObjects().getValue().getUsedVariables(usedVariables);
-        ILogicalExpression offsetExpr = op.getOffset().getValue();
-        if (offsetExpr != null) {
-            offsetExpr.getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Void arg) {
-        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
-            oe.second.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Void arg) {
-        List<LogicalVariable> parameterVariables = op.getVariables();
-        for (LogicalVariable v : parameterVariables) {
-            if (!usedVariables.contains(v)) {
-                usedVariables.add(v);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
-            exprRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Void arg) {
-        List<LogicalVariable> parameterVariables = op.getInputVariables();
-        for (LogicalVariable v : parameterVariables) {
-            if (!usedVariables.contains(v)) {
-                usedVariables.add(v);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Void arg) {
-        op.getCondition().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Void arg) {
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> m : op.getVariableMappings()) {
-            if (!usedVariables.contains(m.first)) {
-                usedVariables.add(m.first);
-            }
-            if (!usedVariables.contains(m.second)) {
-                usedVariables.add(m.second);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
-        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Void arg) {
-        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
-            expr.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
-            expr.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) {
-        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
-        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) {
-        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Void arg) {
-        return null;
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
-        op.getDelegate().getUsedVariables(usedVariables);
-        return null;
-    }
-
-}
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class UsedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+    private Collection<LogicalVariable> usedVariables;
+
+    public UsedVariableVisitor(Collection<LogicalVariable> usedVariables) {
+        this.usedVariables = usedVariables;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+            eRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+        // Used variables depend on the physical operator.
+        if (op.getPhysicalOperator() != null) {
+            IPhysicalOperator physOp = op.getPhysicalOperator();
+            switch (physOp.getOperatorTag()) {
+                case BROADCAST_EXCHANGE:
+                case ONE_TO_ONE_EXCHANGE:
+                case RANDOM_MERGE_EXCHANGE: {
+                    // No variables used.
+                    break;
+                }
+                case HASH_PARTITION_EXCHANGE: {
+                    HashPartitionExchangePOperator concreteOp = (HashPartitionExchangePOperator) physOp;
+                    usedVariables.addAll(concreteOp.getHashFields());
+                    break;
+                }
+                case HASH_PARTITION_MERGE_EXCHANGE: {
+                    HashPartitionMergeExchangePOperator concreteOp = (HashPartitionMergeExchangePOperator) physOp;
+                    usedVariables.addAll(concreteOp.getPartitionFields());
+                    for (OrderColumn orderCol : concreteOp.getOrderColumns()) {
+                        usedVariables.add(orderCol.getColumn());
+                    }
+                    break;
+                }
+                case SORT_MERGE_EXCHANGE: {
+                    SortMergeExchangePOperator concreteOp = (SortMergeExchangePOperator) physOp;
+                    for (OrderColumn orderCol : concreteOp.getSortColumns()) {
+                        usedVariables.add(orderCol.getColumn());
+                    }
+                    break;
+                }
+                case RANGE_PARTITION_EXCHANGE: {
+                    RangePartitionPOperator concreteOp = (RangePartitionPOperator) physOp;
+                    for (OrderColumn partCol : concreteOp.getPartitioningFields()) {
+                        usedVariables.add(partCol.getColumn());
+                    }
+                    break;
+                }
+                default: {
+                    throw new AlgebricksException("Unhandled physical operator tag '" + physOp.getOperatorTag() + "'.");
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getGroupByList()) {
+            g.second.getValue().getUsedVariables(usedVariables);
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getDecorList()) {
+            g.second.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) {
+        op.getCondition().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) {
+        op.getCondition().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Void arg) {
+        op.getMaxObjects().getValue().getUsedVariables(usedVariables);
+        ILogicalExpression offsetExpr = op.getOffset().getValue();
+        if (offsetExpr != null) {
+            offsetExpr.getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Void arg) {
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+            oe.second.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Void arg) {
+        List<LogicalVariable> parameterVariables = op.getVariables();
+        for (LogicalVariable v : parameterVariables) {
+            if (!usedVariables.contains(v)) {
+                usedVariables.add(v);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Void arg) {
+        List<LogicalVariable> parameterVariables = op.getInputVariables();
+        for (LogicalVariable v : parameterVariables) {
+            if (!usedVariables.contains(v)) {
+                usedVariables.add(v);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Void arg) {
+        op.getCondition().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Void arg) {
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> m : op.getVariableMappings()) {
+            if (!usedVariables.contains(m.first)) {
+                usedVariables.add(m.first);
+            }
+            if (!usedVariables.contains(m.second)) {
+                usedVariables.add(m.second);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Void arg) {
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
+            expr.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
+            expr.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) {
+        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) {
+        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Void arg) {
+        return null;
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+        op.getDelegate().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
index 5a2619d..1ca6d26 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -33,6 +33,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -396,4 +397,13 @@
         sb.append("]");
     }
 
+    @Override
+    public String visitExternalDataLookupOperator(ExternalDataLookupOperator op, Integer indent)
+            throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append(
+                "external-instant-lookup " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
+        return buffer.toString();
+    }
+
 }
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
index 0467d63..3fdee94 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
@@ -23,6 +23,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -104,5 +105,7 @@
     public R visitInsertDeleteOperator(InsertDeleteOperator op, T tag) throws AlgebricksException;
 
     public R visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, T tag) throws AlgebricksException;
+    
+    public R visitExternalDataLookupOperator(ExternalDataLookupOperator op, T arg) throws AlgebricksException;
 
 }
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndex.java
index 4006cb7..5ae96f0 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndex.java
@@ -122,4 +122,10 @@
      */
     public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
             boolean checkIfEmptyIndex) throws IndexException;
+    
+    /**
+     * 
+     * @return true if the index needs memory components
+     */
+    public boolean hasMemoryComponents();
 }
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITwoPCIndexBulkLoader.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITwoPCIndexBulkLoader.java
new file mode 100644
index 0000000..8fda095
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITwoPCIndexBulkLoader.java
@@ -0,0 +1,39 @@
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface ITwoPCIndexBulkLoader {
+    
+    /**
+     * Append a "delete" tuple to the index in the context of a bulk load.
+     * 
+     * @param tuple
+     *            "delete" Tuple to be inserted.
+     * @throws IndexException
+     *             If the input stream is invalid for bulk loading (e.g., is not sorted).
+     * @throws HyracksDataException
+     *             If the BufferCache throws while un/pinning or un/latching.
+     */
+    public void delete(ITupleReference tuple) throws IndexException, HyracksDataException;
+    
+    /**
+     * Abort the bulk modify op
+     */
+    public void abort();
+
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
index a0f9d8e..81c13a1 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
@@ -29,18 +29,18 @@
 import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
 
 public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
-    private final IIndexOperatorDescriptor opDesc;
-    private final IHyracksTaskContext ctx;
-    private final float fillFactor;
-    private final boolean verifyInput;
-    private final long numElementsHint;
-    private final boolean checkIfEmptyIndex;
-    private final IIndexDataflowHelper indexHelper;
-    private FrameTupleAccessor accessor;
-    private IIndex index;
-    private IIndexBulkLoader bulkLoader;
-    private IRecordDescriptorProvider recDescProvider;
-    private PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
+    protected final IIndexOperatorDescriptor opDesc;
+    protected final IHyracksTaskContext ctx;
+    protected final float fillFactor;
+    protected final boolean verifyInput;
+    protected final long numElementsHint;
+    protected final boolean checkIfEmptyIndex;
+    protected final IIndexDataflowHelper indexHelper;
+    protected FrameTupleAccessor accessor;
+    protected IIndex index;
+    protected IIndexBulkLoader bulkLoader;
+    protected IRecordDescriptorProvider recDescProvider;
+    protected PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
 
     public IndexBulkLoadOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
             int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index acc5bc2..f2ff6eb 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -167,7 +167,6 @@
                 }
             }
         }
-
     }
 
     @Override
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 517660b..7fdcd96 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -337,6 +337,14 @@
             interiorFrame.initBuffer((byte) nodeFrontiers.size());
             nodeFrontiers.add(frontier);
         }
+
+        public ITreeIndexFrame getLeafFrame() {
+            return leafFrame;
+        }
+
+        public void setLeafFrame(ITreeIndexFrame leafFrame) {
+            this.leafFrame = leafFrame;
+        }
     }
 
     public class TreeIndexInsertBulkLoader implements IIndexBulkLoader {
@@ -367,4 +375,13 @@
     public long getMemoryAllocationSize() {
         return 0;
     }
+
+    public IBinaryComparatorFactory[] getCmpFactories() {
+        return cmpFactories;
+    }
+    
+    @Override
+    public boolean hasMemoryComponents() {
+        return true;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java
index 9a916d6..896abb5 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java
@@ -23,7 +23,7 @@
 		ITreeIndexTupleWriterFactory {
 
 	private static final long serialVersionUID = 1L;
-	private ITypeTraits[] typeTraits;
+	protected ITypeTraits[] typeTraits;
 
 	public TypeAwareTupleWriterFactory(ITypeTraits[] typeTraits) {
 		this.typeTraits = typeTraits;
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
new file mode 100644
index 0000000..2a458ac
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+
+public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
+
+    private int version;
+
+    public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+            ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version) {
+        super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
+                ioOpCallbackFactory, false);
+        this.version = version;
+    }
+
+    public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+            List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
+            ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version) {
+        this(opDesc, ctx, partition, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory,
+                ioScheduler, ioOpCallbackFactory, needKeyDupCheck, version);
+    }
+
+    @Override
+    public IIndex getIndexInstance() {
+        if (index != null)
+            return index;
+        synchronized (lcManager) {
+            long resourceID;
+            try {
+                resourceID = getResourceID();
+            } catch (HyracksDataException e) {
+                return null;
+            }
+            try {
+                index = lcManager.getIndex(resourceID);
+            } catch (HyracksDataException e) {
+                return null;
+            }
+        }
+        return index;
+    }
+
+    @Override
+    public ITreeIndex createIndexInstance() throws HyracksDataException {
+        AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
+        return LSMBTreeUtils.createExternalBTree(file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
+                .getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
+                .getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
+                bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
+                ioOpCallbackFactory.createIOOperationCallback(), getVersion());
+    }
+
+    public int getVersion() {
+        return version;
+    }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
new file mode 100644
index 0000000..48dd41e
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflowHelperFactory;
+
+public class ExternalBTreeDataflowHelperFactory extends AbstractLSMIndexDataflowHelperFactory {
+
+    private static final long serialVersionUID = 1L;
+
+    private int version;
+
+    public ExternalBTreeDataflowHelperFactory(ILSMMergePolicyFactory mergePolicyFactory,
+            Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
+            ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+            double bloomFilterFalsePositiveRate, int version) {
+        super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
+                ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+        this.version = version;
+    }
+
+    @Override
+    public IIndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+            int partition) {
+        return new ExternalBTreeDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
+                mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
+                ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, false, version);
+    }
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
new file mode 100644
index 0000000..7fb5009
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflowHelper;
+
+public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDataflowHelper {
+
+    private int[] buddyBtreeFields;
+    private int version;
+
+    public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+            int partition, ILSMMergePolicy mergePolicy,
+            ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version) {
+        super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory, ioScheduler,
+                ioOpCallbackFactory);
+        this.buddyBtreeFields = buddyBtreeFields;
+        this.version = version;
+    }
+
+    public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+            int partition, double bloomFilterFalsePositiveRate,
+            ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+            int[] buddyBtreeFields, int version) {
+        super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
+                ioScheduler, ioOpCallbackFactory);
+        this.buddyBtreeFields = buddyBtreeFields;
+        this.version = version;
+    }
+    
+    @Override
+    public IIndex getIndexInstance() {
+        if(index!= null)
+            return index;
+        synchronized (lcManager) {
+            long resourceID;
+            try {
+                resourceID = getResourceID();
+            } catch (HyracksDataException e) {
+                return null;
+            }
+            try {
+                index = lcManager.getIndex(resourceID);
+            } catch (HyracksDataException e) {
+                return null;
+            }
+        }
+        return index;
+    }
+
+    @Override
+    protected IIndex createIndexInstance() throws HyracksDataException {
+        AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
+        return LSMBTreeUtils.createExternalBTreeWithBuddy(file, opDesc.getStorageManager().getBufferCache(ctx),
+                opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
+                treeOpDesc.getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy,
+                opTrackerFactory.getOperationTracker(ctx), ioScheduler,
+                ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, version);
+    }
+    
+    public int getTargetVersion(){
+        return version;
+    }
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
new file mode 100644
index 0000000..67f0c85
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflowHelperFactory;
+
+public class ExternalBTreeWithBuddyDataflowHelperFactory extends AbstractLSMIndexDataflowHelperFactory {
+
+    private static final long serialVersionUID = 1L;
+    private final int[] buddyBtreeFields;
+    private int version;
+
+    public ExternalBTreeWithBuddyDataflowHelperFactory(
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+            ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+            ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
+            int[] buddyBtreeFields, int version) {
+        super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
+                ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+        this.buddyBtreeFields = buddyBtreeFields;
+        this.version = version;
+    }
+
+    @Override
+    public IIndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+            int partition) {
+        return new ExternalBTreeWithBuddyDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
+                mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
+                ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, buddyBtreeFields, version);
+    }
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
new file mode 100644
index 0000000..da19882
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -0,0 +1,700 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+/**
+ * This is an lsm b-tree that does not have memory component and is modified
+ * only by bulk loading and addition of disk components as of this point, it is
+ * intended for use with external dataset indexes only.
+ * 
+ * @author alamouda
+ */
+public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
+
+    // This component factory has to be different since it uses different tuple
+    // writer in it's leaf frames to support inserting both
+    // regular and delete tuples
+    private final LSMBTreeDiskComponentFactory transactionComponentFactory;
+    // A second disk component list that will be used when a transaction is
+    // committed and will be seen by subsequent accessors
+    private List<ILSMComponent> secondDiskComponents;
+    // A pointer that points to the current most recent list (either
+    // diskComponents = 0, or secondDiskComponents = 1). It starts with -1 to
+    // indicate first time activation
+    private int version = -1;
+
+    private final ITreeIndexFrameFactory interiorFrameFactory;
+
+    public ExternalBTree(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+            ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
+            TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
+            BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
+            IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
+            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree> transactionBTreeFactory, int version) {
+        super(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
+                bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider,
+                fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false);
+        this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory);
+        this.secondDiskComponents = new LinkedList<ILSMComponent>();
+        this.interiorFrameFactory = interiorFrameFactory;
+        this.version = version;
+    }
+
+    // This method is used to create a target for a bulk modify operation. This
+    // component must then be either committed or deleted
+    private ILSMComponent createTransactionTarget() throws HyracksDataException, IndexException {
+        LSMComponentFileReferences componentFileRefs;
+        try {
+            componentFileRefs = fileManager.getNewTransactionFileReference();
+        } catch (IOException e) {
+            throw new HyracksDataException("Failed to create transaction components", e);
+        }
+        return createDiskComponent(transactionComponentFactory, componentFileRefs.getInsertIndexFileReference(),
+                componentFileRefs.getBloomFilterFileReference(), true);
+    }
+
+    // The subsume merged components is overridden to account for:
+    // Maintaining two versions of the index
+    @Override
+    public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+            throws HyracksDataException {
+        List<ILSMComponent> newerList;
+        List<ILSMComponent> olderList;
+        if (version == 0) {
+            newerList = diskComponents;
+            olderList = secondDiskComponents;
+        } else {
+            newerList = secondDiskComponents;
+            olderList = diskComponents;
+        }
+
+        // Check if merge will affect the older list
+        if (olderList.containsAll(mergedComponents)) {
+            int swapIndex = olderList.indexOf(mergedComponents.get(0));
+            olderList.removeAll(mergedComponents);
+            olderList.add(swapIndex, newComponent);
+        }
+
+        // The new list will always have all the merged components
+        int swapIndex = newerList.indexOf(mergedComponents.get(0));
+        newerList.removeAll(mergedComponents);
+        newerList.add(swapIndex, newComponent);
+    }
+
+    // This method is used by the merge policy when it needs to check if a merge
+    // is needed.
+    // It only needs to return the newer list
+    @Override
+    public List<ILSMComponent> getImmutableComponents() {
+        if (version == 0) {
+            return diskComponents;
+        } else {
+            return secondDiskComponents;
+        }
+    }
+
+    // The only reason to override the following method is that it uses a different context object
+    @Override
+    public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+            throws HyracksDataException, IndexException {
+        ExternalBTreeOpContext ctx = (ExternalBTreeOpContext) ictx;
+        List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+        LSMBTreeCursorInitialState initialState = new LSMBTreeCursorInitialState(insertLeafFrameFactory, ctx.cmp,
+                ctx.bloomFilterCmp, lsmHarness, pred, ctx.searchCallback, operationalComponents);
+        cursor.open(initialState, pred);
+    }
+
+    // The only reason to override the following method is that it uses a different context object
+    // in addition, determining whether or not to keep deleted tuples is different here
+    @Override
+    public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException, IndexException {
+        ExternalBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, -1);
+        opCtx.setOperation(IndexOperation.MERGE);
+        List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+        boolean returnDeletedTuples = false;
+        if (version == 0) {
+            if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != diskComponents.get(diskComponents
+                    .size() - 1)) {
+                returnDeletedTuples = true;
+            }
+        } else {
+            if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != secondDiskComponents
+                    .get(secondDiskComponents.size() - 1)) {
+                returnDeletedTuples = true;
+            }
+        }
+        ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
+        BTree firstBTree = (BTree) ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
+        BTree lastBTree = (BTree) ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1))
+                .getBTree();
+        FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
+        FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
+        LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile()
+                .getName(), lastFile.getFile().getName());
+        ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
+        ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs
+                .getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager
+                .getBaseDir()));
+    }
+
+    // This function should only be used when a transaction fail. it doesn't
+    // take any parameters since there can only be
+    // a single transaction and hence a single transaction component on disk
+    public void deleteTransactionComponent() throws HyracksDataException {
+        fileManager.deleteTransactionFiles();
+    }
+
+    // This function in an instance of this index is only used after a bulk load
+    // is successful
+    @Override
+    public void addComponent(ILSMComponent c) throws HyracksDataException {
+        if (version == 0) {
+            diskComponents.add(0, c);
+        } else if (version == 1) {
+            secondDiskComponents.add(0, c);
+        }
+    }
+
+    // This function is used when a new component is to be committed.
+    public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
+        // determine which list is the new one and flip the pointer
+        List<ILSMComponent> newerList;
+        List<ILSMComponent> olderList;
+        if (version == 0) {
+            newerList = diskComponents;
+            olderList = secondDiskComponents;
+            version = 1;
+        } else {
+            newerList = secondDiskComponents;
+            olderList = diskComponents;
+            version = 0;
+        }
+        // Remove components from list
+        olderList.clear();
+        // Add components
+        olderList.addAll(newerList);
+        if (newComponent != null) {
+            // Add it to the list
+            olderList.add(0, newComponent);
+        }
+    }
+
+    @Override
+    public synchronized void activate() throws HyracksDataException {
+        if (isActivated) {
+            throw new HyracksDataException("Failed to activate the index since it is already activated.");
+        }
+        if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
+            //First time activation
+            List<LSMComponentFileReferences> validFileReferences;
+            try {
+                validFileReferences = fileManager.cleanupAndGetValidFiles();
+            } catch (IndexException e) {
+                throw new HyracksDataException(e);
+            }
+            for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
+                LSMBTreeDiskComponent component;
+                try {
+                    component = createDiskComponent(componentFactory,
+                            lsmComonentFileReference.getInsertIndexFileReference(),
+                            lsmComonentFileReference.getBloomFilterFileReference(), false);
+                } catch (IndexException e) {
+                    throw new HyracksDataException(e);
+                }
+                diskComponents.add(component);
+                secondDiskComponents.add(component);
+            }
+            ((ExternalIndexHarness) lsmHarness).indexFirstTimeActivated();
+        } else {
+            // This index has been opened before
+            for (ILSMComponent c : diskComponents) {
+                LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+                BTree btree = component.getBTree();
+                BloomFilter bloomFilter = component.getBloomFilter();
+                btree.activate();
+                bloomFilter.activate();
+            }
+            for (ILSMComponent c : secondDiskComponents) {
+                // Only activate non shared components
+                if (!diskComponents.contains(c)) {
+                    LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+                    BTree btree = component.getBTree();
+                    BloomFilter bloomFilter = component.getBloomFilter();
+                    btree.activate();
+                    bloomFilter.activate();
+                }
+            }
+        }
+        isActivated = true;
+    }
+
+    @Override
+    public synchronized void create() throws HyracksDataException {
+        super.create();
+        secondDiskComponents.clear();
+    }
+
+    @Override
+    public void deactivate(boolean flushOnExit) throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
+        }
+        if (flushOnExit) {
+            BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
+            cb.afterFinalize(LSMOperationType.FLUSH, null);
+        }
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            BTree btree = component.getBTree();
+            BloomFilter bloomFilter = component.getBloomFilter();
+            btree.deactivate();
+            bloomFilter.deactivate();
+        }
+        for (ILSMComponent c : secondDiskComponents) {
+            // Only deactivate non shared components (So components are not de-activated twice)
+            if (!diskComponents.contains(c)) {
+                LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+                BTree btree = component.getBTree();
+                BloomFilter bloomFilter = component.getBloomFilter();
+                btree.deactivate();
+                bloomFilter.deactivate();
+            }
+        }
+        isActivated = false;
+    }
+
+    // The clear method is not used anywhere in AsterixDB! we override it anyway
+    // to exit components first and deal with the two lists
+    @Override
+    public void clear() throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to clear the index since it is not activated.");
+        }
+        ((ExternalIndexHarness) lsmHarness).indexClear();
+
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            component.getBloomFilter().deactivate();
+            component.getBTree().deactivate();
+            component.getBloomFilter().destroy();
+            component.getBTree().destroy();
+            // Remove from second list to avoid destroying twice
+            secondDiskComponents.remove(c);
+        }
+
+        for (ILSMComponent c : secondDiskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            component.getBloomFilter().deactivate();
+            component.getBTree().deactivate();
+            component.getBloomFilter().destroy();
+            component.getBTree().destroy();
+        }
+        diskComponents.clear();
+        secondDiskComponents.clear();
+        version = -1;
+    }
+
+    @Override
+    public void destroy() throws HyracksDataException {
+        if (isActivated) {
+            throw new HyracksDataException("Failed to destroy the index since it is activated.");
+        }
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
+            // Remove from second list to avoid destroying twice
+            secondDiskComponents.remove(c);
+        }
+        for (ILSMComponent c : secondDiskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
+        }
+        diskComponents.clear();
+        secondDiskComponents.clear();
+        fileManager.deleteDirs();
+        version = -1;
+    }
+
+    @Override
+    public void validate() throws HyracksDataException {
+        for (ILSMComponent c : diskComponents) {
+            BTree btree = (BTree) ((LSMBTreeDiskComponent) c).getBTree();
+            btree.validate();
+        }
+        for (ILSMComponent c : secondDiskComponents) {
+            if (!diskComponents.contains(c)) {
+                BTree btree = (BTree) ((LSMBTreeDiskComponent) c).getBTree();
+                btree.validate();
+            }
+        }
+    }
+
+    // Not supported
+    @Override
+    public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+        throw new UnsupportedOperationException("tuple modify not supported in LSM-Disk-Only-BTree");
+    }
+
+    // Not supported
+    @Override
+    public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException {
+        throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-BTree");
+    }
+
+    // Not supported
+    @Override
+    public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+        throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-BTree");
+    }
+
+    // Only support search and merge operations
+    @Override
+    public void getOperationalComponents(ILSMIndexOperationContext ctx) {
+        List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+        List<ILSMComponent> immutableComponents;
+        // Identify current list in case of a merge
+        if (version == 0) {
+            immutableComponents = diskComponents;
+        } else {
+            immutableComponents = secondDiskComponents;
+        }
+        ExternalBTreeOpContext opCtx = (ExternalBTreeOpContext) ctx;
+        operationalComponents.clear();
+        switch (ctx.getOperation()) {
+            case SEARCH:
+                if (opCtx.getTargetIndexVersion() == 0) {
+                    operationalComponents.addAll(diskComponents);
+                } else {
+                    operationalComponents.addAll(secondDiskComponents);
+                }
+                break;
+            case MERGE:
+                operationalComponents.addAll(ctx.getComponentsToBeMerged());
+                break;
+            case FULL_MERGE:
+                operationalComponents.addAll(immutableComponents);
+                break;
+            default:
+                throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
+        }
+    }
+
+    // For initial load
+    @Override
+    public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex) throws TreeIndexException {
+        try {
+            return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, false);
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    // For transaction bulk load <- could consolidate with the above method ->
+    @Override
+    public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex) throws TreeIndexException {
+        try {
+            return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, true);
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    // The bulk loader used for both initial loading and transaction
+    // modifications
+    public class LSMTwoPCBTreeBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
+        private final ILSMComponent component;
+        private final BTreeBulkLoader bulkLoader;
+        private final IIndexBulkLoader builder;
+        private boolean cleanedUpArtifacts = false;
+        private boolean isEmptyComponent = true;
+        private boolean endedBloomFilterLoad = false;
+        private boolean isTransaction;
+        private ITreeIndexTupleWriterFactory frameTupleWriterFactory;
+
+        public LSMTwoPCBTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+                boolean checkIfEmptyIndex, boolean isTransaction) throws TreeIndexException, HyracksDataException {
+            this.isTransaction = isTransaction;
+            // Create the appropriate target
+            if (isTransaction) {
+                try {
+                    component = createTransactionTarget();
+                } catch (HyracksDataException | IndexException e) {
+                    throw new TreeIndexException(e);
+                }
+            } else {
+                if (checkIfEmptyIndex && !isEmptyIndex()) {
+                    throw new TreeIndexException("Cannot load an index that is not empty");
+                }
+                try {
+                    component = createBulkLoadTarget();
+                } catch (HyracksDataException | IndexException e) {
+                    throw new TreeIndexException(e);
+                }
+            }
+
+            frameTupleWriterFactory = ((LSMBTreeDiskComponent) component).getBTree().getLeafFrameFactory()
+                    .getTupleWriterFactory();
+            bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
+                    verifyInput, numElementsHint, false);
+
+            int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
+            BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+                    bloomFilterFalsePositiveRate);
+            builder = ((LSMBTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
+                    bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+        }
+
+        // It is expected that the mode was set to insert operation before
+        // calling add
+        @Override
+        public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
+            try {
+                bulkLoader.add(tuple);
+                builder.add(tuple);
+            } catch (IndexException | HyracksDataException | RuntimeException e) {
+                cleanupArtifacts();
+                throw e;
+            }
+            if (isEmptyComponent) {
+                isEmptyComponent = false;
+            }
+        }
+
+        // This is made public in case of a failure, it is better to delete all
+        // created artifacts.
+        public void cleanupArtifacts() throws HyracksDataException, IndexException {
+            if (!cleanedUpArtifacts) {
+                cleanedUpArtifacts = true;
+                // We make sure to end the bloom filter load to release latches.
+                if (!endedBloomFilterLoad) {
+                    builder.end();
+                    endedBloomFilterLoad = true;
+                }
+                try {
+                    ((LSMBTreeDiskComponent) component).getBTree().deactivate();
+                } catch (HyracksDataException e) {
+                    // Do nothing.. this could've bee
+                }
+                ((LSMBTreeDiskComponent) component).getBTree().destroy();
+                try {
+                    ((LSMBTreeDiskComponent) component).getBloomFilter().deactivate();
+                } catch (HyracksDataException e) {
+                    // Do nothing.. this could've bee
+                }
+                ((LSMBTreeDiskComponent) component).getBloomFilter().destroy();
+            }
+        }
+
+        @Override
+        public void end() throws HyracksDataException, IndexException {
+            if (!cleanedUpArtifacts) {
+                if (!endedBloomFilterLoad) {
+                    builder.end();
+                    endedBloomFilterLoad = true;
+                }
+                bulkLoader.end();
+                if (isEmptyComponent) {
+                    cleanupArtifacts();
+                } else if (isTransaction) {
+                    // Since this is a transaction component, validate and
+                    // deactivate. it could later be added or deleted
+                    markAsValid(component);
+                    BTree btree = ((LSMBTreeDiskComponent) component).getBTree();
+                    BloomFilter bloomFilter = ((LSMBTreeDiskComponent) component).getBloomFilter();
+                    btree.deactivate();
+                    bloomFilter.deactivate();
+                } else {
+                    lsmHarness.addBulkLoadedComponent(component);
+                }
+            }
+        }
+
+        // It is expected that the mode was set to delete operation before
+        // calling delete
+        @Override
+        public void delete(ITupleReference tuple) throws IndexException, HyracksDataException {
+            ((LSMBTreeRefrencingTupleWriterFactory) frameTupleWriterFactory).setMode(IndexOperation.DELETE);
+            try {
+                bulkLoader.add(tuple);
+                builder.add(tuple);
+            } catch (IndexException | HyracksDataException | RuntimeException e) {
+                cleanupArtifacts();
+                throw e;
+            }
+            if (isEmptyComponent) {
+                isEmptyComponent = false;
+            }
+            ((LSMBTreeRefrencingTupleWriterFactory) frameTupleWriterFactory).setMode(IndexOperation.INSERT);
+        }
+
+        @Override
+        public void abort() {
+            try{
+                cleanupArtifacts();
+            } catch(Exception e){
+                // Do nothing
+            }
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "LSMTwoPCBTree [" + fileManager.getBaseDir() + "]";
+    }
+
+    // The accessor for disk only indexes don't use modification callback and always carry the target index version with them
+    @Override
+    public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+            ISearchOperationCallback searchCallback) {
+        return new LSMBTreeAccessor(lsmHarness, createOpContext(searchCallback, version));
+    }
+
+    // This method creates the appropriate opContext for the targeted version
+    public ExternalBTreeOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
+        return new ExternalBTreeOpContext(insertLeafFrameFactory, deleteLeafFrameFactory, searchCallback,
+                componentFactory.getBloomFilterKeyFields().length, cmpFactories, targetVersion);
+    }
+
+    @Override
+    public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
+            throws HyracksDataException {
+        return new LSMBTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
+    }
+
+    @Override
+    public ITreeIndexFrameFactory getInteriorFrameFactory() {
+        return interiorFrameFactory;
+    }
+
+    @Override
+    public int getFieldCount() {
+        return cmpFactories.length;
+    }
+
+    @Override
+    public int getFileId() {
+        return -1;
+    }
+
+    @Override
+    public IFreePageManager getFreePageManager() {
+        return null;
+    }
+
+    @Override
+    public ITreeIndexFrameFactory getLeafFrameFactory() {
+        return insertLeafFrameFactory;
+    }
+
+    public int getCurrentVersion() {
+        return version;
+    }
+
+    public List<ILSMComponent> getFirstComponentList() {
+        return diskComponents;
+    }
+
+    public List<ILSMComponent> getSecondComponentList() {
+        return secondDiskComponents;
+    }
+
+    @Override
+    public void commitTransaction() throws TreeIndexException, HyracksDataException, IndexException {
+        LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
+        LSMBTreeDiskComponent component = null;
+        if (componentFileRefrences != null) {
+            component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(),
+                    componentFileRefrences.getBloomFilterFileReference(), false);
+        }
+        ((ExternalIndexHarness) lsmHarness).addTransactionComponents(component);
+    }
+
+    @Override
+    public void abortTransaction() throws TreeIndexException {
+        try {
+            fileManager.deleteTransactionFiles();
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    @Override
+    public void recoverTransaction() throws TreeIndexException {
+        try {
+            fileManager.recoverTransaction();
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+    
+    @Override
+    public boolean hasMemoryComponents(){
+        return false;
+    }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
new file mode 100644
index 0000000..220943f
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class ExternalBTreeOpContext implements ILSMIndexOperationContext {
+	public ITreeIndexFrameFactory insertLeafFrameFactory;
+	public ITreeIndexFrameFactory deleteLeafFrameFactory;
+	public IBTreeLeafFrame insertLeafFrame;
+	public IBTreeLeafFrame deleteLeafFrame;
+	public IndexOperation op;
+	public final MultiComparator cmp;
+	public final MultiComparator bloomFilterCmp;
+	public final ISearchOperationCallback searchCallback;
+	private final List<ILSMComponent> componentHolder;
+	private final List<ILSMComponent> componentsToBeMerged;
+	private final int targetIndexVersion;
+
+	public ExternalBTreeOpContext(
+			ITreeIndexFrameFactory insertLeafFrameFactory,
+			ITreeIndexFrameFactory deleteLeafFrameFactory,
+			ISearchOperationCallback searchCallback,
+			int numBloomFilterKeyFields,
+			IBinaryComparatorFactory[] cmpFactories, int targetIndexVersion) {
+		if (cmpFactories != null) {
+			this.cmp = MultiComparator.create(cmpFactories);
+		} else {
+			this.cmp = null;
+		}
+		bloomFilterCmp = MultiComparator.create(cmpFactories, 0,
+				numBloomFilterKeyFields);
+		this.insertLeafFrameFactory = insertLeafFrameFactory;
+		this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+		this.insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory
+				.createFrame();
+		this.deleteLeafFrame = (IBTreeLeafFrame) deleteLeafFrameFactory
+				.createFrame();
+		if (insertLeafFrame != null && this.cmp != null) {
+			insertLeafFrame.setMultiComparator(cmp);
+		}
+		if (deleteLeafFrame != null && this.cmp != null) {
+			deleteLeafFrame.setMultiComparator(cmp);
+		}
+		this.componentHolder = new LinkedList<ILSMComponent>();
+		this.componentsToBeMerged = new LinkedList<ILSMComponent>();
+		this.searchCallback = searchCallback;
+		this.targetIndexVersion = targetIndexVersion;
+	}
+
+	@Override
+	public void setOperation(IndexOperation newOp) {
+		reset();
+		this.op = newOp;
+	}
+
+	@Override
+	public void reset() {
+		componentHolder.clear();
+		componentsToBeMerged.clear();
+	}
+
+	public IndexOperation getOperation() {
+		return op;
+	}
+
+	@Override
+	public List<ILSMComponent> getComponentHolder() {
+		return componentHolder;
+	}
+
+	@Override
+	public ISearchOperationCallback getSearchOperationCallback() {
+		return searchCallback;
+	}
+
+	// Disk only index should never needs a modification callback
+	@Override
+	public IModificationOperationCallback getModificationCallback() {
+		return null;
+	}
+
+	@Override
+	public void setCurrentMutableComponentId(int currentMutableComponentId) {
+		// Do nothing: this method should never be called for this class
+	}
+
+	@Override
+	public List<ILSMComponent> getComponentsToBeMerged() {
+		return componentsToBeMerged;
+	}
+
+	// Used by indexes with global transaction
+	public int getTargetIndexVersion() {
+		return targetIndexVersion;
+	}
+
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
new file mode 100644
index 0000000..f5fac99
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -0,0 +1,873 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeIndex, ITwoPCIndex {
+
+    // For creating merge disk components
+    private final LSMBTreeWithBuddyDiskComponentFactory componentFactory;
+
+    private final LSMBTreeWithBuddyDiskComponentFactory bulkComponentFactory;
+
+    private final IBinaryComparatorFactory[] btreeCmpFactories;
+    private final IBinaryComparatorFactory[] buddyBtreeCmpFactories;
+    private int[] buddyBTreeFields;
+
+    // Common for in-memory and on-disk components.
+    private final ITreeIndexFrameFactory btreeInteriorFrameFactory;
+    private final ITreeIndexFrameFactory btreeLeafFrameFactory;
+    private final ITreeIndexFrameFactory buddyBtreeLeafFrameFactory;
+
+    // A second disk component list that will be used when a transaction is
+    // committed and will be seen by subsequent accessors
+    private List<ILSMComponent> secondDiskComponents;
+    private int version = -1;
+
+    public ExternalBTreeWithBuddy(ITreeIndexFrameFactory btreeInteriorFrameFactory,
+            ITreeIndexFrameFactory btreeLeafFrameFactory, ITreeIndexFrameFactory buddyBtreeLeafFrameFactory,
+            IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
+            TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory,
+            TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory,
+            IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
+            int[] buddyBTreeFields, int version) {
+        super(diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
+                ioScheduler, ioOpCallback);
+        this.btreeCmpFactories = btreeCmpFactories;
+        this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
+        this.buddyBTreeFields = buddyBTreeFields;
+        this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
+        this.btreeLeafFrameFactory = btreeLeafFrameFactory;
+        this.buddyBtreeLeafFrameFactory = buddyBtreeLeafFrameFactory;
+        this.componentFactory = new LSMBTreeWithBuddyDiskComponentFactory(copyBtreeFactory, buddyBtreeFactory,
+                bloomFilterFactory);
+        this.bulkComponentFactory = new LSMBTreeWithBuddyDiskComponentFactory(bulkLoadBTreeFactory, buddyBtreeFactory,
+                bloomFilterFactory);
+        this.secondDiskComponents = new LinkedList<ILSMComponent>();
+        this.version = version;
+    }
+
+    @Override
+    public void create() throws HyracksDataException {
+        if (isActivated) {
+            throw new HyracksDataException("Failed to create the index since it is activated.");
+        }
+        fileManager.deleteDirs();
+        fileManager.createDirs();
+        diskComponents.clear();
+        secondDiskComponents.clear();
+    }
+
+    @Override
+    public void activate() throws HyracksDataException {
+        if (isActivated) {
+            throw new HyracksDataException("Failed to activate the index since it is already activated.");
+        }
+
+        if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
+            //First time activation
+            List<LSMComponentFileReferences> validFileReferences;
+            try {
+                validFileReferences = fileManager.cleanupAndGetValidFiles();
+            } catch (IndexException e) {
+                throw new HyracksDataException(e);
+            }
+            for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
+                LSMBTreeWithBuddyDiskComponent component;
+                try {
+                    component = createDiskComponent(componentFactory,
+                            lsmComonentFileReference.getInsertIndexFileReference(),
+                            lsmComonentFileReference.getDeleteIndexFileReference(),
+                            lsmComonentFileReference.getBloomFilterFileReference(), false);
+                } catch (IndexException e) {
+                    throw new HyracksDataException(e);
+                }
+                diskComponents.add(component);
+                secondDiskComponents.add(component);
+            }
+            ((ExternalIndexHarness) lsmHarness).indexFirstTimeActivated();
+        } else {
+            // This index has been opened before or is brand new with no
+            // components. It should also maintain the version pointer
+            for (ILSMComponent c : diskComponents) {
+                LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+                BTree btree = component.getBTree();
+                BTree buddyBtree = component.getBuddyBTree();
+                BloomFilter bloomFilter = component.getBloomFilter();
+                btree.activate();
+                buddyBtree.activate();
+                bloomFilter.activate();
+            }
+            for (ILSMComponent c : secondDiskComponents) {
+                // Only activate non shared components
+                if (!diskComponents.contains(c)) {
+                    LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+                    BTree btree = component.getBTree();
+                    BTree buddyBtree = component.getBuddyBTree();
+                    BloomFilter bloomFilter = component.getBloomFilter();
+                    btree.activate();
+                    buddyBtree.activate();
+                    bloomFilter.activate();
+                }
+            }
+        }
+        isActivated = true;
+    }
+
+    @Override
+    public void clear() throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to clear the index since it is not activated.");
+        }
+        ((ExternalIndexHarness) lsmHarness).indexClear();
+
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+            component.getBTree().deactivate();
+            component.getBuddyBTree().deactivate();
+            component.getBloomFilter().deactivate();
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
+            component.getBuddyBTree().destroy();
+            // Remove from second list to avoid destroying twice
+            secondDiskComponents.remove(c);
+        }
+
+        for (ILSMComponent c : secondDiskComponents) {
+            LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+            component.getBTree().deactivate();
+            component.getBloomFilter().deactivate();
+            component.getBuddyBTree().deactivate();
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
+            component.getBuddyBTree().destroy();
+        }
+
+        diskComponents.clear();
+        secondDiskComponents.clear();
+        version = -1;
+    }
+
+    @Override
+    public void deactivate() throws HyracksDataException {
+        deactivate(true);
+    }
+
+    @Override
+    public void destroy() throws HyracksDataException {
+        if (isActivated) {
+            throw new HyracksDataException("Failed to destroy the index since it is activated.");
+        }
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+            component.getBTree().destroy();
+            component.getBuddyBTree().destroy();
+            component.getBloomFilter().destroy();
+            // Remove from second list to avoid destroying twice
+            secondDiskComponents.remove(c);
+        }
+        for (ILSMComponent c : secondDiskComponents) {
+            LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+            component.getBTree().destroy();
+            component.getBuddyBTree().destroy();
+            component.getBloomFilter().destroy();
+        }
+        diskComponents.clear();
+        secondDiskComponents.clear();
+        fileManager.deleteDirs();
+        version = -1;
+    }
+
+    @Override
+    public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+            ISearchOperationCallback searchCallback) throws HyracksDataException {
+        return new LSMBTreeWithBuddyAccessor(lsmHarness, createOpContext(searchCallback, version));
+    }
+
+    @Override
+    public void validate() throws HyracksDataException {
+        throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
+    }
+
+    @Override
+    public long getMemoryAllocationSize() {
+        return 0;
+    }
+
+    // The subsume merged components is overridden to account for:
+    // Maintaining two versions of the index
+    @Override
+    public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+            throws HyracksDataException {
+        List<ILSMComponent> newerList;
+        List<ILSMComponent> olderList;
+        if (version == 0) {
+            newerList = diskComponents;
+            olderList = secondDiskComponents;
+        } else {
+            newerList = secondDiskComponents;
+            olderList = diskComponents;
+        }
+
+        // Check if merge will affect the older list
+        if (olderList.containsAll(mergedComponents)) {
+            int swapIndex = olderList.indexOf(mergedComponents.get(0));
+            olderList.removeAll(mergedComponents);
+            olderList.add(swapIndex, newComponent);
+        }
+
+        // The new list will always have all the merged components
+        int swapIndex = newerList.indexOf(mergedComponents.get(0));
+        newerList.removeAll(mergedComponents);
+        newerList.add(swapIndex, newComponent);
+    }
+
+    // For initial load
+    @Override
+    public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex) throws TreeIndexException {
+        try {
+            return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex,
+                    false);
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    // For transaction bulk load <- could consolidate with the above method ->
+    public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex) throws TreeIndexException {
+        try {
+            return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex,
+                    true);
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    @Override
+    public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+        throw new UnsupportedOperationException("tuple modify not supported in LSM-Disk-Only-BTree");
+    }
+
+    @Override
+    public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+            throws HyracksDataException, IndexException {
+        ExternalBTreeWithBuddyOpContext ctx = (ExternalBTreeWithBuddyOpContext) ictx;
+        List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
+
+        LSMBTreeWithBuddyCursorInitialState initialState = new LSMBTreeWithBuddyCursorInitialState(
+                btreeInteriorFrameFactory, btreeLeafFrameFactory, buddyBtreeLeafFrameFactory, lsmHarness,
+                MultiComparator.create(btreeCmpFactories), MultiComparator.create(buddyBtreeCmpFactories),
+                ctx.searchCallback, operationalComponents);
+
+        cursor.open(initialState, pred);
+    }
+
+    @Override
+    public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException {
+        throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-BTree");
+    }
+
+    @Override
+    public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+        throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-BTree");
+    }
+
+    protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
+            throws HyracksDataException {
+        BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getBTree();
+        BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1))
+                .getBTree();
+        FileReference firstFile = diskFileMapProvider.lookupFileName(firstTree.getFileId());
+        FileReference lastFile = diskFileMapProvider.lookupFileName(lastTree.getFileId());
+        LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(),
+                lastFile.getFile().getName());
+        return fileRefs;
+    }
+
+    @Override
+    public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException, IndexException {
+        ILSMIndexOperationContext bctx = createOpContext(NoOpOperationCallback.INSTANCE, 0);
+        bctx.setOperation(IndexOperation.MERGE);
+        List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+        ITreeIndexCursor cursor = new LSMBTreeWithBuddySortedCursor(bctx, buddyBTreeFields);
+        LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
+        ILSMIndexAccessorInternal accessor = new LSMBTreeWithBuddyAccessor(lsmHarness, bctx);
+
+        // Since we have two lists of components, to tell whether we need to
+        // keep deleted tuples, we need to know
+        // which list to check against and we need to synchronize for this
+        boolean keepDeleteTuples = false;
+        if (version == 0) {
+            keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents
+                    .size() - 1);
+        } else {
+            keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != secondDiskComponents
+                    .get(secondDiskComponents.size() - 1);
+        }
+
+        ioScheduler.scheduleOperation(new LSMBTreeWithBuddyMergeOperation((ILSMIndexAccessorInternal) accessor,
+                mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs
+                        .getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback,
+                fileManager.getBaseDir(), keepDeleteTuples));
+    }
+
+    // This method creates the appropriate opContext for the targeted version
+    public ExternalBTreeWithBuddyOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
+        return new ExternalBTreeWithBuddyOpContext(btreeCmpFactories, buddyBtreeCmpFactories, searchCallback,
+                targetVersion);
+    }
+
+    @Override
+    public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+        LSMBTreeWithBuddyMergeOperation mergeOp = (LSMBTreeWithBuddyMergeOperation) operation;
+        ITreeIndexCursor cursor = mergeOp.getCursor();
+        ISearchPredicate btreeSearchPred = new RangePredicate(null, null, true, true, null, null);
+        ILSMIndexOperationContext opCtx = ((LSMBTreeWithBuddySortedCursor) cursor).getOpCtx();
+        opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
+        search(opCtx, cursor, btreeSearchPred);
+
+        LSMBTreeWithBuddyDiskComponent mergedComponent = createDiskComponent(componentFactory,
+                mergeOp.getBTreeMergeTarget(), mergeOp.getBuddyBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(),
+                true);
+
+        // In case we must keep the deleted-keys BuddyBTrees, then they must be
+        // merged *before* merging the b-trees so that
+        // lsmHarness.endSearch() is called once when the b-trees have been
+        // merged.
+
+        if (mergeOp.isKeepDeletedTuples()) {
+            // Keep the deleted tuples since the oldest disk component is not
+            // included in the merge operation
+            LSMBuddyBTreeMergeCursor buddyBtreeCursor = new LSMBuddyBTreeMergeCursor(opCtx);
+            search(opCtx, buddyBtreeCursor, btreeSearchPred);
+
+            BTree buddyBtree = mergedComponent.getBuddyBTree();
+            IIndexBulkLoader buddyBtreeBulkLoader = buddyBtree.createBulkLoader(1.0f, true, 0L, false);
+
+            long numElements = 0L;
+            for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+                numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i))
+                        .getBloomFilter().getNumElements();
+            }
+
+            int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
+            BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+                    bloomFilterFalsePositiveRate);
+            IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
+                    bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+
+            try {
+                while (buddyBtreeCursor.hasNext()) {
+                    buddyBtreeCursor.next();
+                    ITupleReference tuple = buddyBtreeCursor.getTuple();
+                    buddyBtreeBulkLoader.add(tuple);
+                    builder.add(tuple);
+                }
+            } finally {
+                buddyBtreeCursor.close();
+                builder.end();
+            }
+            buddyBtreeBulkLoader.end();
+        }
+
+        IIndexBulkLoader bulkLoader = mergedComponent.getBTree().createBulkLoader(1.0f, false, 0L, false);
+        try {
+            while (cursor.hasNext()) {
+                cursor.next();
+                ITupleReference frameTuple = cursor.getTuple();
+                bulkLoader.add(frameTuple);
+            }
+        } finally {
+            cursor.close();
+        }
+        bulkLoader.end();
+        return mergedComponent;
+    }
+
+    @Override
+    public void getOperationalComponents(ILSMIndexOperationContext ctx) {
+        List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+        List<ILSMComponent> immutableComponents;
+        // Identify current list in case of a merge
+        if (version == 0) {
+            immutableComponents = diskComponents;
+        } else {
+            immutableComponents = secondDiskComponents;
+        }
+
+        ExternalBTreeWithBuddyOpContext opCtx = (ExternalBTreeWithBuddyOpContext) ctx;
+        operationalComponents.clear();
+        switch (ctx.getOperation()) {
+            case SEARCH:
+                if (opCtx.getTargetIndexVersion() == 0) {
+                    operationalComponents.addAll(diskComponents);
+                } else {
+                    operationalComponents.addAll(secondDiskComponents);
+                }
+                break;
+            case MERGE:
+                operationalComponents.addAll(ctx.getComponentsToBeMerged());
+                break;
+            case FULL_MERGE:
+                operationalComponents.addAll(immutableComponents);
+                break;
+            case FLUSH:
+                // Do nothing. this is left here even though the index never
+                // performs flushes because a flush is triggered by
+                // dataset lifecycle manager when closing an index. Having no
+                // components is a no operation
+                break;
+            default:
+                throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
+        }
+    }
+
+    @Override
+    public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
+        LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
+        // Flush the bloom filter first.
+        int fileId = component.getBloomFilter().getFileId();
+        IBufferCache bufferCache = component.getBTree().getBufferCache();
+        int startPage = 0;
+        int maxPage = component.getBloomFilter().getNumPages();
+        forceFlushDirtyPages(bufferCache, fileId, startPage, maxPage);
+        forceFlushDirtyPages(component.getBTree());
+        markAsValidInternal(component.getBTree());
+        forceFlushDirtyPages(component.getBuddyBTree());
+        markAsValidInternal(component.getBuddyBTree());
+    }
+
+    // This function is used when a new component is to be committed -- is
+    // called by the harness.
+    public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
+
+        // determine which list is the new one and flip the pointer
+        List<ILSMComponent> newerList;
+        List<ILSMComponent> olderList;
+        if (version == 0) {
+            newerList = diskComponents;
+            olderList = secondDiskComponents;
+            version = 1;
+        } else {
+            newerList = secondDiskComponents;
+            olderList = diskComponents;
+            version = 0;
+        }
+        // Remove components from list
+        olderList.clear();
+        // Add components
+        olderList.addAll(newerList);
+        if (newComponent != null) {
+            // Add it to the list
+            olderList.add(0, newComponent);
+        }
+    }
+
+    @Override
+    public void deactivate(boolean flushOnExit) throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
+        }
+
+        if (flushOnExit) {
+            BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
+            cb.afterFinalize(LSMOperationType.FLUSH, null);
+        }
+        // Even though, we deactivate the index, we don't exit components or
+        // modify any of the lists to make sure they
+        // are there if the index was opened again
+
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+            BTree btree = component.getBTree();
+            BTree buddyBtree = component.getBuddyBTree();
+            BloomFilter bloomFilter = component.getBloomFilter();
+            btree.deactivate();
+            buddyBtree.deactivate();
+            bloomFilter.deactivate();
+        }
+        for (ILSMComponent c : secondDiskComponents) {
+            // Only deactivate non shared components
+            if (!diskComponents.contains(c)) {
+                LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+                BTree btree = component.getBTree();
+                BTree buddyBtree = component.getBuddyBTree();
+                BloomFilter bloomFilter = component.getBloomFilter();
+                btree.deactivate();
+                buddyBtree.deactivate();
+                bloomFilter.deactivate();
+            }
+        }
+        isActivated = false;
+    }
+
+    @Override
+    public ITreeIndexFrameFactory getLeafFrameFactory() {
+        return btreeLeafFrameFactory;
+    }
+
+    @Override
+    public ITreeIndexFrameFactory getInteriorFrameFactory() {
+        return btreeInteriorFrameFactory;
+    }
+
+    @Override
+    public IFreePageManager getFreePageManager() {
+        // This method should never be called for disk only indexes
+        return null;
+    }
+
+    @Override
+    public int getFieldCount() {
+        return btreeCmpFactories.length;
+    }
+
+    @Override
+    public int getRootPageId() {
+        // This method should never be called for this index
+        return 0;
+    }
+
+    @Override
+    public int getFileId() {
+        // This method should never be called for this index
+        return 0;
+    }
+
+    @Override
+    public IBinaryComparatorFactory[] getComparatorFactories() {
+        return btreeCmpFactories;
+    }
+
+    private LSMBTreeWithBuddyDiskComponent createDiskComponent(ILSMComponentFactory factory,
+            FileReference insertFileRef, FileReference deleteFileRef, FileReference bloomFilterFileRef,
+            boolean createComponent) throws HyracksDataException, IndexException {
+        // Create new instance.
+        LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory
+                .createLSMComponentInstance(new LSMComponentFileReferences(insertFileRef, deleteFileRef,
+                        bloomFilterFileRef));
+        if (createComponent) {
+            component.getBTree().create();
+            component.getBuddyBTree().create();
+            component.getBloomFilter().create();
+        }
+
+        component.getBTree().activate();
+        component.getBuddyBTree().activate();
+        component.getBloomFilter().activate();
+        return component;
+    }
+
+    // even though the index doesn't support record level modification, the
+    // accessor will try to do it
+    // we could throw the exception here but we don't. it will eventually be
+    // thrown by the index itself
+    public class LSMBTreeWithBuddyAccessor extends LSMTreeIndexAccessor {
+        public LSMBTreeWithBuddyAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
+            super(lsmHarness, ctx);
+        }
+
+        @Override
+        public ITreeIndexCursor createSearchCursor(boolean exclusive) {
+            return new LSMBTreeWithBuddySearchCursor(ctx, buddyBTreeFields);
+        }
+
+        public MultiComparator getBTreeMultiComparator() {
+            ExternalBTreeWithBuddyOpContext concreteCtx = (ExternalBTreeWithBuddyOpContext) ctx;
+            return concreteCtx.getBTreeMultiComparator();
+        }
+
+        public MultiComparator getBodyBTreeMultiComparator() {
+            ExternalBTreeWithBuddyOpContext concreteCtx = (ExternalBTreeWithBuddyOpContext) ctx;
+            return concreteCtx.getBuddyBTreeMultiComparator();
+        }
+    }
+
+    // The bulk loader used for both initial loading and transaction
+    // modifications
+    public class LSMTwoPCBTreeWithBuddyBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
+        private final ILSMComponent component;
+        private final BTreeBulkLoader btreeBulkLoader;
+        private final BTreeBulkLoader buddyBtreeBulkLoader;
+        private final IIndexBulkLoader builder;
+        private boolean cleanedUpArtifacts = false;
+        private boolean isEmptyComponent = true;
+        private boolean endedBloomFilterLoad = false;
+        private boolean isTransaction;
+
+        public LSMTwoPCBTreeWithBuddyBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+                boolean checkIfEmptyIndex, boolean isTransaction) throws TreeIndexException, HyracksDataException {
+            this.isTransaction = isTransaction;
+            // Create the appropriate target
+            if (isTransaction) {
+                try {
+                    component = createTransactionTarget();
+                } catch (HyracksDataException | IndexException e) {
+                    throw new TreeIndexException(e);
+                }
+            } else {
+                if (checkIfEmptyIndex && !isEmptyIndex()) {
+                    throw new TreeIndexException("Cannot load an index that is not empty");
+                }
+                try {
+                    component = createBulkLoadTarget();
+                } catch (HyracksDataException | IndexException e) {
+                    throw new TreeIndexException(e);
+                }
+            }
+
+            // Create the three loaders
+            btreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBTree()
+                    .createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
+            buddyBtreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree()
+                    .createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
+            int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
+            BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+                    bloomFilterFalsePositiveRate);
+            builder = ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
+                    bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+        }
+
+        @Override
+        public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
+            try {
+                btreeBulkLoader.add(tuple);
+            } catch (IndexException | HyracksDataException | RuntimeException e) {
+                cleanupArtifacts();
+                throw e;
+            }
+            if (isEmptyComponent) {
+                isEmptyComponent = false;
+            }
+        }
+
+        // This is made public in case of a failure, it is better to delete all
+        // created artifacts.
+        public void cleanupArtifacts() throws HyracksDataException {
+            if (!cleanedUpArtifacts) {
+                cleanedUpArtifacts = true;
+                try {
+                    ((LSMBTreeWithBuddyDiskComponent) component).getBTree().deactivate();
+                } catch (Exception e) {
+
+                }
+                ((LSMBTreeWithBuddyDiskComponent) component).getBTree().destroy();
+                try {
+                    ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree().deactivate();
+                } catch (Exception e) {
+
+                }
+                ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree().destroy();
+                try {
+                    ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter().deactivate();
+                } catch (Exception e) {
+
+                }
+                ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter().destroy();
+            }
+        }
+
+        @Override
+        public void end() throws HyracksDataException, IndexException {
+            if (!cleanedUpArtifacts) {
+                if (!endedBloomFilterLoad) {
+                    builder.end();
+                    endedBloomFilterLoad = true;
+                }
+                btreeBulkLoader.end();
+                buddyBtreeBulkLoader.end();
+                if (isEmptyComponent) {
+                    cleanupArtifacts();
+                } else if (isTransaction) {
+                    // Since this is a transaction component, validate and
+                    // deactivate. it could later be added or deleted
+                    markAsValid(component);
+                    BTree btree = ((LSMBTreeWithBuddyDiskComponent) component).getBTree();
+                    BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree();
+                    BloomFilter bloomFilter = ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter();
+                    btree.deactivate();
+                    buddyBtree.deactivate();
+                    bloomFilter.deactivate();
+                } else {
+                    lsmHarness.addBulkLoadedComponent(component);
+                }
+            }
+        }
+
+        @Override
+        public void delete(ITupleReference tuple) throws IndexException, HyracksDataException {
+            try {
+                buddyBtreeBulkLoader.add(tuple);
+                builder.add(tuple);
+            } catch (IndexException | HyracksDataException | RuntimeException e) {
+                cleanupArtifacts();
+                throw e;
+            }
+            if (isEmptyComponent) {
+                isEmptyComponent = false;
+            }
+        }
+
+        @Override
+        public void abort() {
+            try {
+                cleanupArtifacts();
+            } catch (Exception e) {
+            }
+        }
+    }
+
+    protected ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+        LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+        return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
+                componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
+    }
+
+    // This method is used to create a target for a bulk modify operation. This
+    // component must then eventually be either committed or deleted
+    private ILSMComponent createTransactionTarget() throws HyracksDataException, IndexException {
+        LSMComponentFileReferences componentFileRefs;
+        try {
+            componentFileRefs = fileManager.getNewTransactionFileReference();
+        } catch (IOException e) {
+            throw new HyracksDataException("Failed to create transaction components", e);
+        }
+        return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
+                componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
+    }
+
+    @Override
+    public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
+            throws HyracksDataException {
+        return new LSMBTreeWithBuddyAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
+    }
+
+    // This function in an instance of this index is only used after a bulk load
+    // is successful
+    // it will therefore add the component to the first list and enter it.
+    @Override
+    public void addComponent(ILSMComponent c) throws HyracksDataException {
+        if (version == 0) {
+            diskComponents.add(0, c);
+        } else if (version == 1) {
+            secondDiskComponents.add(0, c);
+        }
+    }
+
+    public int getCurrentVersion() {
+        return version;
+    }
+
+    public List<ILSMComponent> getFirstComponentList() {
+        return diskComponents;
+    }
+
+    public List<ILSMComponent> getSecondComponentList() {
+        return secondDiskComponents;
+    }
+
+    @Override
+    public void commitTransaction() throws HyracksDataException, IndexException {
+        LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
+        LSMBTreeWithBuddyDiskComponent component = null;
+        if (componentFileRefrences != null) {
+            component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(),
+                    componentFileRefrences.getDeleteIndexFileReference(),
+                    componentFileRefrences.getBloomFilterFileReference(), false);
+        }
+        ((ExternalIndexHarness) lsmHarness).addTransactionComponents(component);
+    }
+
+    @Override
+    public void abortTransaction() throws TreeIndexException {
+        try {
+            fileManager.deleteTransactionFiles();
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    @Override
+    public void recoverTransaction() throws TreeIndexException {
+        try {
+            fileManager.recoverTransaction();
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+    
+    @Override
+    public boolean hasMemoryComponents(){
+        return false;
+    }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
new file mode 100644
index 0000000..e676bc8
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class ExternalBTreeWithBuddyOpContext implements
+		ILSMIndexOperationContext {
+	private IndexOperation op;
+	private MultiComparator bTreeCmp;
+	private MultiComparator buddyBTreeCmp;
+	public final List<ILSMComponent> componentHolder;
+	private final List<ILSMComponent> componentsToBeMerged;
+	public final ISearchOperationCallback searchCallback;
+	private final int targetIndexVersion;
+
+	public ExternalBTreeWithBuddyOpContext(
+			IBinaryComparatorFactory[] btreeCmpFactories,
+			IBinaryComparatorFactory[] buddyBtreeCmpFactories,
+			ISearchOperationCallback searchCallback, int targetIndexVersion) {
+
+		this.componentHolder = new LinkedList<ILSMComponent>();
+		this.componentsToBeMerged = new LinkedList<ILSMComponent>();
+		this.searchCallback = searchCallback;
+		this.targetIndexVersion = targetIndexVersion;
+		this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
+		this.buddyBTreeCmp = MultiComparator.create(buddyBtreeCmpFactories);
+	}
+
+	public void setOperation(IndexOperation newOp) {
+		reset();
+		this.op = newOp;
+	}
+
+	@Override
+	public void setCurrentMutableComponentId(int currentMutableComponentId) {
+		// Do nothing. this should never be called for disk only indexes
+	}
+
+	@Override
+	public void reset() {
+		componentHolder.clear();
+		componentsToBeMerged.clear();
+	}
+
+	@Override
+	public IndexOperation getOperation() {
+		return op;
+	}
+
+	public MultiComparator getBTreeMultiComparator() {
+		return bTreeCmp;
+	}
+
+	public MultiComparator getBuddyBTreeMultiComparator() {
+		return buddyBTreeCmp;
+	}
+
+	@Override
+	public List<ILSMComponent> getComponentHolder() {
+		return componentHolder;
+	}
+
+	@Override
+	public ISearchOperationCallback getSearchOperationCallback() {
+		return searchCallback;
+	}
+
+	// This should never be needed for disk only indexes
+	@Override
+	public IModificationOperationCallback getModificationCallback() {
+		return null;
+	}
+
+	@Override
+	public List<ILSMComponent> getComponentsToBeMerged() {
+		return componentsToBeMerged;
+	}
+
+	public int getTargetIndexVersion() {
+		return targetIndexVersion;
+	}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 19307e9..9a0e315 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -75,17 +75,17 @@
 public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
 
     // For creating BTree's used in flush and merge.
-    private final LSMBTreeDiskComponentFactory componentFactory;
+    protected final LSMBTreeDiskComponentFactory componentFactory;
     // For creating BTree's used in bulk load. Different from diskBTreeFactory
     // because it should have a different tuple writer in it's leaf frames.
-    private final LSMBTreeDiskComponentFactory bulkLoadComponentFactory;
+    protected final LSMBTreeDiskComponentFactory bulkLoadComponentFactory;
 
     // Common for in-memory and on-disk components.
-    private final ITreeIndexFrameFactory insertLeafFrameFactory;
-    private final ITreeIndexFrameFactory deleteLeafFrameFactory;
-    private final IBinaryComparatorFactory[] cmpFactories;
+    protected final ITreeIndexFrameFactory insertLeafFrameFactory;
+    protected final ITreeIndexFrameFactory deleteLeafFrameFactory;
+    protected final IBinaryComparatorFactory[] cmpFactories;
 
-    private final boolean needKeyDupCheck;
+    protected final boolean needKeyDupCheck;
 
     public LSMBTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory,
             ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
@@ -115,6 +115,24 @@
         this.needKeyDupCheck = needKeyDupCheck;
     }
 
+    // Without memory components
+    public LSMBTree(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+            ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
+            TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
+            BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
+            IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
+            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck) {
+        super(diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate,
+                mergePolicy, opTracker, ioScheduler, ioOpCallback);
+        this.insertLeafFrameFactory = insertLeafFrameFactory;
+        this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+        this.cmpFactories = cmpFactories;
+        componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory);
+        bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory);
+        this.needKeyDupCheck = needKeyDupCheck;
+    }
+
     @Override
     public synchronized void create() throws HyracksDataException {
         if (isActivated) {
@@ -489,8 +507,9 @@
         return mergedComponent;
     }
 
-    private LSMBTreeDiskComponent createDiskComponent(LSMBTreeDiskComponentFactory factory, FileReference btreeFileRef,
-            FileReference bloomFilterFileRef, boolean createComponent) throws HyracksDataException, IndexException {
+    protected LSMBTreeDiskComponent createDiskComponent(LSMBTreeDiskComponentFactory factory,
+            FileReference btreeFileRef, FileReference bloomFilterFileRef, boolean createComponent)
+            throws HyracksDataException, IndexException {
         // Create new BTree instance.
         LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) factory
                 .createLSMComponentInstance(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
@@ -514,7 +533,7 @@
         }
     }
 
-    private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+    protected ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
         LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
         return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(),
                 componentFileRefs.getBloomFilterFileReference(), true);
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 7e8fbb3..2f39e70 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -17,6 +17,9 @@
 
 import java.io.File;
 import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -78,16 +81,19 @@
         ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
         ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
 
+        // create transaction filter <to hide transaction files>
+        FilenameFilter transactionFilter = getTransactionFileFilter(false);
+
         // Gather files
 
         // List of valid BTree files.
-        cleanupAndGetValidFilesInternal(btreeFilter, btreeFactory, allBTreeFiles);
+        cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
         HashSet<String> btreeFilesSet = new HashSet<String>();
         for (ComparableFileName cmpFileName : allBTreeFiles) {
             int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
             btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
         }
-        validateFiles(btreeFilesSet, allBloomFilterFiles, bloomFilterFilter, null);
+        validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(transactionFilter, bloomFilterFilter), null);
 
         // Sanity check.
         if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
@@ -159,4 +165,59 @@
 
         return validFiles;
     }
+
+    @Override
+    public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
+        Date date = new Date();
+        String ts = formatter.format(date);
+        // Create transaction lock file
+        Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
+
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
+        // Begin timestamp and end timestamp are identical since it is a transaction
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
+                createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
+    }
+
+    @Override
+    public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
+        FilenameFilter transactionFilter;
+        File dir = new File(baseDir);
+        String[] files = dir.list(transactionFileNameFilter);
+        if (files.length == 0) {
+            return null;
+        }
+        if (files.length != 1) {
+            throw new HyracksDataException("More than one transaction lock found:" + files.length);
+        } else {
+            transactionFilter = getTransactionFileFilter(true);
+            String txnFileName = dir.getPath() + File.separator + files[0];
+            // get the actual transaction files
+            files = dir.list(transactionFilter);
+            if (files.length < 2) {
+                throw new HyracksDataException("LSM Btree transaction has less than 2 files :" + files.length);
+            }
+            try {
+                Files.delete(Paths.get(txnFileName));
+            } catch (IOException e) {
+                throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName);
+            }
+        }
+        File bTreeFile = null;
+        File bloomFilterFile = null;
+
+        for (String fileName : files) {
+            if (fileName.endsWith(BTREE_STRING)) {
+                bTreeFile = new File(dir.getPath() + File.separator + fileName);
+            } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
+                bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
+            } else {
+                throw new HyracksDataException("unrecognized file found = " + fileName);
+            }
+        }
+        FileReference bTreeFileRef = new FileReference(bTreeFile);
+        FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+
+        return new LSMComponentFileReferences(bTreeFileRef, null, bloomFilterFileRef);
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
new file mode 100644
index 0000000..6316480
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BloomFilterAwareBTreePointSearchCursor;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public abstract class LSMBTreeWithBuddyAbstractCursor implements
+		ITreeIndexCursor {
+
+	protected boolean open;
+	protected ITreeIndexCursor[] btreeCursors;
+	protected ITreeIndexCursor[] buddyBtreeCursors;
+	protected ITreeIndexAccessor[] btreeAccessors;
+	protected ITreeIndexAccessor[] buddyBtreeAccessors;
+	protected MultiComparator btreeCmp;
+	protected MultiComparator buddyBtreeCmp;
+	protected int numberOfTrees;
+	protected RangePredicate btreeRangePredicate;
+	protected RangePredicate buddyBtreeRangePredicate;
+	protected ITupleReference frameTuple;
+	protected boolean includeMutableComponent;
+	protected ILSMHarness lsmHarness;
+	protected boolean foundNext;
+	protected final ILSMIndexOperationContext opCtx;
+
+	protected List<ILSMComponent> operationalComponents;
+
+	public LSMBTreeWithBuddyAbstractCursor(ILSMIndexOperationContext opCtx) {
+		super();
+		this.opCtx = opCtx;
+	}
+
+	public ITreeIndexCursor getCursor(int cursorIndex) {
+		return btreeCursors[cursorIndex];
+	}
+
+	@Override
+	public void open(ICursorInitialState initialState,
+			ISearchPredicate searchPred) throws IndexException,
+			HyracksDataException {
+
+		LSMBTreeWithBuddyCursorInitialState lsmInitialState = (LSMBTreeWithBuddyCursorInitialState) initialState;
+		btreeCmp = lsmInitialState.getBTreeCmp();
+		buddyBtreeCmp = lsmInitialState.getBuddyBTreeCmp();
+
+		operationalComponents = lsmInitialState.getOperationalComponents();
+		lsmHarness = lsmInitialState.getLSMHarness();
+		numberOfTrees = operationalComponents.size();
+
+		btreeCursors = new ITreeIndexCursor[numberOfTrees];
+		buddyBtreeCursors = new ITreeIndexCursor[numberOfTrees];
+		btreeAccessors = new ITreeIndexAccessor[numberOfTrees];
+		buddyBtreeAccessors = new ITreeIndexAccessor[numberOfTrees];
+
+		includeMutableComponent = false;
+
+		for (int i = 0; i < numberOfTrees; i++) {
+			ILSMComponent component = operationalComponents.get(i);
+			BTree btree;
+			BTree buddyBtree;
+			if (component.getType() == LSMComponentType.MEMORY) {
+				// This is not needed at the moment but is implemented anyway
+				includeMutableComponent = true;
+				// No need for a bloom filter for the in-memory BTree.
+				buddyBtreeCursors[i] = new BTreeRangeSearchCursor(
+						(IBTreeLeafFrame) lsmInitialState
+								.getBuddyBTreeLeafFrameFactory().createFrame(),
+						false);
+				btree = ((LSMBTreeWithBuddyMemoryComponent) component)
+						.getBTree();
+				buddyBtree = ((LSMBTreeWithBuddyMemoryComponent) component)
+						.getBuddyBTree();
+			} else {
+				buddyBtreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor(
+						(IBTreeLeafFrame) lsmInitialState
+								.getBuddyBTreeLeafFrameFactory().createFrame(),
+						false,
+						((LSMBTreeWithBuddyDiskComponent) operationalComponents
+								.get(i)).getBloomFilter());
+				btree = ((LSMBTreeWithBuddyDiskComponent) component).getBTree();
+				buddyBtree = (BTree) ((LSMBTreeWithBuddyDiskComponent) component)
+						.getBuddyBTree();
+			}
+			IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState
+					.getBTreeLeafFrameFactory().createFrame();
+			btreeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
+			btreeAccessors[i] = btree.createAccessor(
+					NoOpOperationCallback.INSTANCE,
+					NoOpOperationCallback.INSTANCE);
+			buddyBtreeAccessors[i] = buddyBtree.createAccessor(
+					NoOpOperationCallback.INSTANCE,
+					NoOpOperationCallback.INSTANCE);
+		}
+		btreeRangePredicate = (RangePredicate) searchPred;
+		buddyBtreeRangePredicate = new RangePredicate(null, null, true, true,
+				buddyBtreeCmp, buddyBtreeCmp);
+
+		open = true;
+	}
+
+	@Override
+	public void close() throws HyracksDataException {
+		if (!open) {
+			return;
+		}
+		try {
+			if (btreeCursors != null && buddyBtreeCursors != null) {
+				for (int i = 0; i < numberOfTrees; i++) {
+					btreeCursors[i].close();
+					buddyBtreeCursors[i].close();
+				}
+			}
+			btreeCursors = null;
+			buddyBtreeCursors = null;
+		} finally {
+			lsmHarness.endSearch(opCtx);
+		}
+		foundNext = false;
+		open = false;
+	}
+
+	@Override
+	public ITupleReference getTuple() {
+		return frameTuple;
+	}
+
+	@Override
+	public ICachedPage getPage() {
+		// Do nothing
+		return null;
+	}
+
+	@Override
+	public void setBufferCache(IBufferCache bufferCache) {
+		// Do nothing
+	}
+
+	@Override
+	public void setFileId(int fileId) {
+		// Do nothing
+	}
+
+	@Override
+	public boolean exclusiveLatchNodes() {
+		return false;
+	}
+
+	@Override
+	public void markCurrentTupleAsUpdated() throws HyracksDataException {
+		throw new HyracksDataException(
+				"Updating tuples is not supported with this cursor.");
+	}
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyCursorInitialState.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyCursorInitialState.java
new file mode 100644
index 0000000..402dddb
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyCursorInitialState.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMBTreeWithBuddyCursorInitialState implements ICursorInitialState {
+	private final ITreeIndexFrameFactory btreeInteriorFrameFactory;
+	private final ITreeIndexFrameFactory btreeLeafFrameFactory;
+	private final ITreeIndexFrameFactory buddyBtreeLeafFrameFactory;
+	private MultiComparator btreeCmp;
+	private MultiComparator buddyBtreeCmp;
+	private final ILSMHarness lsmHarness;
+
+	private ISearchOperationCallback searchCallback;
+	private final List<ILSMComponent> operationalComponents;
+
+	public LSMBTreeWithBuddyCursorInitialState(
+			ITreeIndexFrameFactory btreeInteriorFrameFactory,
+			ITreeIndexFrameFactory btreeLeafFrameFactory,
+			ITreeIndexFrameFactory buddyBtreeLeafFrameFactory,
+			ILSMHarness lsmHarness,
+			MultiComparator btreeCmp, MultiComparator buddyBtreeCmp,
+			ISearchOperationCallback searchCallback,
+			List<ILSMComponent> operationalComponents) {
+		this.btreeLeafFrameFactory = btreeLeafFrameFactory;
+		this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
+		this.buddyBtreeLeafFrameFactory = buddyBtreeLeafFrameFactory;
+		this.btreeCmp = btreeCmp;
+		this.buddyBtreeCmp = buddyBtreeCmp;
+		this.lsmHarness = lsmHarness;
+		this.searchCallback = searchCallback;
+		this.operationalComponents = operationalComponents;
+	}
+
+	public ITreeIndexFrameFactory getBTreeInteriorFrameFactory() {
+		return btreeInteriorFrameFactory;
+	}
+
+	public ITreeIndexFrameFactory getBTreeLeafFrameFactory() {
+		return btreeLeafFrameFactory;
+	}
+
+	public ITreeIndexFrameFactory getBuddyBTreeLeafFrameFactory() {
+		return buddyBtreeLeafFrameFactory;
+	}
+
+	public MultiComparator getBTreeCmp() {
+		return btreeCmp;
+	}
+
+	public MultiComparator getBuddyBTreeCmp() {
+		return buddyBtreeCmp;
+	}
+
+	public List<ILSMComponent> getOperationalComponents() {
+		return operationalComponents;
+	}
+
+	public ILSMHarness getLSMHarness() {
+		return lsmHarness;
+	}
+
+	@Override
+	public ICachedPage getPage() {
+		return null;
+	}
+
+	@Override
+	public void setPage(ICachedPage page) {
+	}
+
+	@Override
+	public ISearchOperationCallback getSearchOperationCallback() {
+		return searchCallback;
+	}
+
+	@Override
+	public void setSearchOperationCallback(
+			ISearchOperationCallback searchCallback) {
+		this.searchCallback = searchCallback;
+	}
+
+	@Override
+	public MultiComparator getOriginalKeyComparator() {
+		return btreeCmp;
+	}
+
+	@Override
+	public void setOriginialKeyComparator(MultiComparator originalCmp) {
+		this.btreeCmp = originalCmp;
+	}
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
new file mode 100644
index 0000000..85063de
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
+
+public class LSMBTreeWithBuddyDiskComponent extends AbstractDiskLSMComponent {
+
+	private final BTree btree;
+	private final BTree buddyBtree;
+	private final BloomFilter bloomFilter;
+
+	public LSMBTreeWithBuddyDiskComponent(BTree btree, BTree buddyBtree,
+			BloomFilter bloomFilter) {
+		this.btree = btree;
+		this.buddyBtree = buddyBtree;
+		this.bloomFilter = bloomFilter;
+	}
+
+	@Override
+	protected void destroy() throws HyracksDataException {
+		btree.deactivate();
+		btree.destroy();
+		buddyBtree.deactivate();
+		buddyBtree.destroy();
+		bloomFilter.deactivate();
+		bloomFilter.destroy();
+	}
+
+	public BTree getBTree() {
+		return btree;
+	}
+
+	public BTree getBuddyBTree() {
+		return buddyBtree;
+	}
+
+	public BloomFilter getBloomFilter() {
+		return bloomFilter;
+	}
+
+	@Override
+	public long getComponentSize() {
+		long size = btree.getFileReference().getFile().length();
+		size += buddyBtree.getFileReference().getFile().length();
+		size += bloomFilter.getFileReference().getFile().length();
+		return size;
+	}
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
new file mode 100644
index 0000000..aa6256a
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class LSMBTreeWithBuddyDiskComponentFactory implements
+		ILSMComponentFactory {
+
+	private final TreeIndexFactory<BTree> btreeFactory;
+	private final TreeIndexFactory<BTree> buddyBtreeFactory;
+	private final BloomFilterFactory bloomFilterFactory;
+
+	public LSMBTreeWithBuddyDiskComponentFactory(
+			TreeIndexFactory<BTree> btreeFactory,
+			TreeIndexFactory<BTree> buddyBtreeFactory,
+			BloomFilterFactory bloomFilterFactory) {
+		this.btreeFactory = btreeFactory;
+		this.buddyBtreeFactory = buddyBtreeFactory;
+		this.bloomFilterFactory = bloomFilterFactory;
+	}
+
+	@Override
+	public ILSMComponent createLSMComponentInstance(
+			LSMComponentFileReferences cfr) throws IndexException,
+			HyracksDataException {
+		return new LSMBTreeWithBuddyDiskComponent(
+				btreeFactory.createIndexInstance(cfr
+						.getInsertIndexFileReference()),
+				buddyBtreeFactory.createIndexInstance(cfr
+						.getDeleteIndexFileReference()),
+				bloomFilterFactory.createBloomFiltertInstance(cfr
+						.getBloomFilterFileReference()));
+	}
+
+	@Override
+	public IBufferCache getBufferCache() {
+		return btreeFactory.getBufferCache();
+	}
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
new file mode 100644
index 0000000..51ee0fb
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -0,0 +1,257 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
+    private static final String BUDDY_BTREE_STRING = "buddy";
+    private static final String BTREE_STRING = "b";
+
+    private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
+    private final TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory;
+
+    private static FilenameFilter btreeFilter = new FilenameFilter() {
+        public boolean accept(File dir, String name) {
+            return !name.startsWith(".") && name.endsWith(BTREE_STRING);
+        }
+    };
+
+    private static FilenameFilter buddyBtreeFilter = new FilenameFilter() {
+        public boolean accept(File dir, String name) {
+            return !name.startsWith(".") && name.endsWith(BUDDY_BTREE_STRING);
+        }
+    };
+
+    public LSMBTreeWithBuddyFileManager(IFileMapProvider fileMapProvider, FileReference file,
+            TreeIndexFactory<? extends ITreeIndex> btreeFactory,
+            TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory) {
+        super(fileMapProvider, file, null);
+        this.buddyBtreeFactory = buddyBtreeFactory;
+        this.btreeFactory = btreeFactory;
+    }
+
+    @Override
+    public LSMComponentFileReferences getRelFlushFileReference() {
+        Date date = new Date();
+        String ts = formatter.format(date);
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
+        // Begin timestamp and end timestamp are identical since it is a flush
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+                createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
+                        + BLOOM_FILTER_STRING));
+    }
+
+    @Override
+    public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
+            throws HyracksDataException {
+        String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+        String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
+
+        String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
+        // Get the range of timestamps by taking the earliest and the latest
+        // timestamps
+        return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
+                createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createMergeFile(baseName + SPLIT_STRING
+                        + BLOOM_FILTER_STRING));
+    }
+
+    @Override
+    public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
+        List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
+        ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
+        ArrayList<ComparableFileName> allBuddyBTreeFiles = new ArrayList<ComparableFileName>();
+        ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
+
+        // Create transaction file filter
+        FilenameFilter transactionFilefilter = getTransactionFileFilter(false);
+
+        // Gather files.
+        cleanupAndGetValidFilesInternal(getCompoundFilter(btreeFilter, transactionFilefilter), btreeFactory,
+                allBTreeFiles);
+        HashSet<String> btreeFilesSet = new HashSet<String>();
+        for (ComparableFileName cmpFileName : allBTreeFiles) {
+            int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
+            btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
+        }
+        validateFiles(btreeFilesSet, allBuddyBTreeFiles, getCompoundFilter(buddyBtreeFilter, transactionFilefilter),
+                buddyBtreeFactory);
+        validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(bloomFilterFilter, transactionFilefilter),
+                null);
+
+        // Sanity check.
+        if (allBTreeFiles.size() != allBuddyBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) {
+            throw new HyracksDataException(
+                    "Unequal number of valid BTree, Buddy BTree, and Bloom Filter files found. Aborting cleanup.");
+        }
+
+        // Trivial cases.
+        if (allBTreeFiles.isEmpty() || allBuddyBTreeFiles.isEmpty() || allBloomFilterFiles.isEmpty()) {
+            return validFiles;
+        }
+
+        if (allBTreeFiles.size() == 1 && allBuddyBTreeFiles.size() == 1 && allBloomFilterFiles.size() == 1) {
+            validFiles.add(new LSMComponentFileReferences(allBTreeFiles.get(0).fileRef,
+                    allBuddyBTreeFiles.get(0).fileRef, allBloomFilterFiles.get(0).fileRef));
+            return validFiles;
+        }
+
+        // Sorts files names from earliest to latest timestamp.
+        Collections.sort(allBTreeFiles);
+        Collections.sort(allBuddyBTreeFiles);
+        Collections.sort(allBloomFilterFiles);
+
+        List<ComparableFileName> validComparableBTreeFiles = new ArrayList<ComparableFileName>();
+        ComparableFileName lastBTree = allBTreeFiles.get(0);
+        validComparableBTreeFiles.add(lastBTree);
+
+        List<ComparableFileName> validComparableBuddyBTreeFiles = new ArrayList<ComparableFileName>();
+        ComparableFileName lastBuddyBTree = allBuddyBTreeFiles.get(0);
+        validComparableBuddyBTreeFiles.add(lastBuddyBTree);
+
+        List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<ComparableFileName>();
+        ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
+        validComparableBloomFilterFiles.add(lastBloomFilter);
+
+        for (int i = 1; i < allBTreeFiles.size(); i++) {
+            ComparableFileName currentBTree = allBTreeFiles.get(i);
+            ComparableFileName currentBuddyBTree = allBuddyBTreeFiles.get(i);
+            ComparableFileName currentBloomFilter = allBloomFilterFiles.get(i);
+            // Current start timestamp is greater than last stop timestamp.
+            if (currentBTree.interval[0].compareTo(lastBTree.interval[1]) > 0
+                    && currentBuddyBTree.interval[0].compareTo(lastBuddyBTree.interval[1]) > 0
+                    && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[1]) > 0) {
+                validComparableBTreeFiles.add(currentBTree);
+                validComparableBuddyBTreeFiles.add(currentBuddyBTree);
+                validComparableBloomFilterFiles.add(currentBloomFilter);
+                lastBTree = currentBTree;
+                lastBuddyBTree = currentBuddyBTree;
+                lastBloomFilter = currentBloomFilter;
+            } else if (currentBTree.interval[0].compareTo(lastBTree.interval[0]) >= 0
+                    && currentBTree.interval[1].compareTo(lastBTree.interval[1]) <= 0
+                    && currentBuddyBTree.interval[0].compareTo(lastBuddyBTree.interval[0]) >= 0
+                    && currentBuddyBTree.interval[1].compareTo(lastBuddyBTree.interval[1]) <= 0
+                    && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
+                    && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
+                // Invalid files are completely contained in last interval.
+                File invalidBTreeFile = new File(currentBTree.fullPath);
+                invalidBTreeFile.delete();
+                File invalidBuddyBTreeFile = new File(currentBuddyBTree.fullPath);
+                invalidBuddyBTreeFile.delete();
+                File invalidBloomFilterFile = new File(currentBloomFilter.fullPath);
+                invalidBloomFilterFile.delete();
+            } else {
+                // This scenario should not be possible.
+                throw new HyracksDataException("Found LSM files with overlapping but not contained timetamp intervals.");
+            }
+        }
+
+        // Sort valid files in reverse lexicographical order, such that newer
+        // files come first.
+        Collections.sort(validComparableBTreeFiles, recencyCmp);
+        Collections.sort(validComparableBuddyBTreeFiles, recencyCmp);
+        Collections.sort(validComparableBloomFilterFiles, recencyCmp);
+
+        Iterator<ComparableFileName> btreeFileIter = validComparableBTreeFiles.iterator();
+        Iterator<ComparableFileName> buddyBtreeFileIter = validComparableBuddyBTreeFiles.iterator();
+        Iterator<ComparableFileName> bloomFilterFileIter = validComparableBloomFilterFiles.iterator();
+        while (btreeFileIter.hasNext() && buddyBtreeFileIter.hasNext()) {
+            ComparableFileName cmpBTreeFileName = btreeFileIter.next();
+            ComparableFileName cmpBuddyBTreeFileName = buddyBtreeFileIter.next();
+            ComparableFileName cmpBloomFilterFileName = bloomFilterFileIter.next();
+            validFiles.add(new LSMComponentFileReferences(cmpBTreeFileName.fileRef, cmpBuddyBTreeFileName.fileRef,
+                    cmpBloomFilterFileName.fileRef));
+        }
+
+        return validFiles;
+    }
+
+    @Override
+    public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
+        Date date = new Date();
+        String ts = formatter.format(date);
+        // Create transaction lock file
+        Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
+
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+                createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
+                        + BLOOM_FILTER_STRING));
+    }
+
+    @Override
+    public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
+        FilenameFilter transactionFilter;
+        File dir = new File(baseDir);
+        String[] files = dir.list(transactionFileNameFilter);
+        if (files.length == 0)
+            return null;
+        if (files.length != 1) {
+            throw new HyracksDataException("More than one transaction lock found:" + files.length);
+        } else {
+            transactionFilter = getTransactionFileFilter(true);
+            String txnFileName = dir.getPath() + File.separator + files[0];
+            // get the actual transaction files
+            files = dir.list(transactionFilter);
+            if (files.length < 3) {
+                throw new HyracksDataException("LSM Btree with buddy transaction has less than 3 files :"
+                        + files.length);
+            }
+            try {
+                Files.delete(Paths.get(txnFileName));
+            } catch (IOException e) {
+                throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName);
+            }
+        }
+        File bTreeFile = null;
+        File buddyBTreeFile = null;
+        File bloomFilterFile = null;
+        for (String fileName : files) {
+            if (fileName.endsWith(BTREE_STRING)) {
+                bTreeFile = new File(dir.getPath() + File.separator + fileName);
+            } else if (fileName.endsWith(BUDDY_BTREE_STRING)) {
+                buddyBTreeFile = new File(dir.getPath() + File.separator + fileName);
+            } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
+                bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
+            } else {
+                throw new HyracksDataException("unrecognized file found = " + fileName);
+            }
+        }
+        FileReference bTreeFileRef = new FileReference(bTreeFile);
+        FileReference buddyBTreeFileRef = new FileReference(buddyBTreeFile);
+        FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+        return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef, bloomFilterFileRef);
+    }
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
new file mode 100644
index 0000000..34509a3
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
+
+/*
+ * This class is also not needed at the moment but is implemented anyway
+ */
+public class LSMBTreeWithBuddyMemoryComponent extends
+		AbstractMemoryLSMComponent {
+
+	private final BTree btree;
+	private final BTree buddyBtree;
+
+	public LSMBTreeWithBuddyMemoryComponent(BTree btree, BTree buddyBtree,
+			IVirtualBufferCache vbc, boolean isActive) {
+		super(vbc, isActive);
+		this.btree = btree;
+		this.buddyBtree = buddyBtree;
+	}
+
+	public BTree getBTree() {
+		return btree;
+	}
+
+	public BTree getBuddyBTree() {
+		return buddyBtree;
+	}
+
+	@Override
+	protected void reset() throws HyracksDataException {
+		super.reset();
+		btree.deactivate();
+		btree.destroy();
+		btree.create();
+		btree.activate();
+		buddyBtree.deactivate();
+		buddyBtree.destroy();
+		buddyBtree.create();
+		buddyBtree.activate();
+	}
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
new file mode 100644
index 0000000..4bae2c8
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+
+public class LSMBTreeWithBuddyMergeOperation implements ILSMIOOperation {
+
+	private final ILSMIndexAccessorInternal accessor;
+	private final List<ILSMComponent> mergingComponents;
+	private final ITreeIndexCursor cursor;
+	private final FileReference btreeMergeTarget;
+	private final FileReference buddyBtreeMergeTarget;
+	private final FileReference bloomFilterMergeTarget;
+	private final ILSMIOOperationCallback callback;
+	private final String indexIdentifier;
+	private final boolean keepDeletedTuples;
+
+	public LSMBTreeWithBuddyMergeOperation(ILSMIndexAccessorInternal accessor,
+			List<ILSMComponent> mergingComponents, ITreeIndexCursor cursor,
+			FileReference btreeMergeTarget,
+			FileReference buddyBtreeMergeTarget,
+			FileReference bloomFilterMergeTarget,
+			ILSMIOOperationCallback callback, String indexIdentifier, boolean keepDeletedTuples) {
+		this.accessor = accessor;
+		this.mergingComponents = mergingComponents;
+		this.cursor = cursor;
+		this.btreeMergeTarget = btreeMergeTarget;
+		this.buddyBtreeMergeTarget = buddyBtreeMergeTarget;
+		this.bloomFilterMergeTarget = bloomFilterMergeTarget;
+		this.callback = callback;
+		this.indexIdentifier = indexIdentifier;
+		this.keepDeletedTuples = keepDeletedTuples;
+	}
+
+	@Override
+	public Set<IODeviceHandle> getReadDevices() {
+		Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+		for (ILSMComponent o : mergingComponents) {
+			LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) o;
+			devs.add(component.getBTree().getFileReference().getDeviceHandle());
+
+			devs.add(component.getBuddyBTree().getFileReference()
+					.getDeviceHandle());
+			devs.add(component.getBloomFilter().getFileReference()
+					.getDeviceHandle());
+
+		}
+		return devs;
+	}
+
+	@Override
+	public Set<IODeviceHandle> getWriteDevices() {
+		Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+		devs.add(btreeMergeTarget.getDeviceHandle());
+
+		devs.add(buddyBtreeMergeTarget.getDeviceHandle());
+		devs.add(bloomFilterMergeTarget.getDeviceHandle());
+
+		return devs;
+	}
+
+	@Override
+	public Boolean call() throws HyracksDataException, IndexException {
+		accessor.merge(this);
+		return true;
+	}
+
+	@Override
+	public ILSMIOOperationCallback getCallback() {
+		return callback;
+	}
+
+	@Override
+	public String getIndexUniqueIdentifier() {
+		return indexIdentifier;
+	}
+
+	@Override
+	public LSMIOOpertionType getIOOpertionType() {
+		return LSMIOOpertionType.MERGE;
+	}
+
+	public FileReference getBTreeMergeTarget() {
+		return btreeMergeTarget;
+	}
+
+	public FileReference getBuddyBTreeMergeTarget() {
+		return buddyBtreeMergeTarget;
+	}
+
+	public FileReference getBloomFilterMergeTarget() {
+		return bloomFilterMergeTarget;
+	}
+
+	public ITreeIndexCursor getCursor() {
+		return cursor;
+	}
+
+	public List<ILSMComponent> getMergingComponents() {
+		return mergingComponents;
+	}
+
+	public boolean isKeepDeletedTuples() {
+		return keepDeletedTuples;
+	}
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySearchCursor.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySearchCursor.java
new file mode 100644
index 0000000..f32bc52
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySearchCursor.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class LSMBTreeWithBuddySearchCursor extends LSMBTreeWithBuddyAbstractCursor{
+	private int currentCursor;
+    private PermutingTupleReference buddyBTreeTuple;
+    
+    public LSMBTreeWithBuddySearchCursor(ILSMIndexOperationContext opCtx, int[] buddyBTreeFields) {
+        super(opCtx);
+        currentCursor = 0;
+        this.buddyBTreeTuple = new PermutingTupleReference(buddyBTreeFields);
+    }
+    
+    @Override
+    public void close() throws HyracksDataException {
+        super.close();
+        currentCursor = 0;
+    }
+
+    @Override
+    public void reset() throws HyracksDataException {
+        if (!open) {
+            return;
+        }
+
+        currentCursor = 0;
+        foundNext = false;
+        try {
+            for (int i = 0; i < numberOfTrees; i++) {
+                btreeCursors[i].close();
+                buddyBtreeCursors[i].close();
+            }
+            btreeCursors = null;
+            buddyBtreeCursors = null;
+        } finally {
+            lsmHarness.endSearch(opCtx);
+        }
+    }
+    
+    private void searchNextCursor() throws HyracksDataException {
+        if (currentCursor < numberOfTrees) {
+            try {
+            	btreeCursors[currentCursor].reset();
+                btreeAccessors[currentCursor].search(btreeCursors[currentCursor], btreeRangePredicate);
+            } catch (IndexException e) {
+                throw new HyracksDataException(e);
+            }
+        }
+    }
+    
+    @Override
+    public boolean hasNext() throws HyracksDataException, IndexException {
+        if (foundNext) {
+            return true;
+        }
+        while (currentCursor < numberOfTrees) {
+            while (btreeCursors[currentCursor].hasNext()) {
+                btreeCursors[currentCursor].next();
+                ITupleReference currentTuple = btreeCursors[currentCursor].getTuple();
+                buddyBTreeTuple.reset(btreeCursors[currentCursor].getTuple());
+                boolean killerTupleFound = false;
+                for (int i = 0; i < currentCursor; i++) {
+                    buddyBtreeCursors[i].reset();
+                    buddyBtreeRangePredicate.setHighKey(buddyBTreeTuple, true);
+                    buddyBtreeRangePredicate.setLowKey(buddyBTreeTuple, true);
+                    buddyBtreeAccessors[i].search(buddyBtreeCursors[i], buddyBtreeRangePredicate);
+                    try {
+                        if (buddyBtreeCursors[i].hasNext()) {
+                            killerTupleFound = true;
+                            break;
+                        }
+                    } finally {
+                        buddyBtreeCursors[i].close();
+                    }
+                }
+                if (!killerTupleFound) {
+                    frameTuple = currentTuple;
+                    foundNext = true;
+                    return true;
+                }
+            }
+            btreeCursors[currentCursor].close();
+            currentCursor++;
+            searchNextCursor();
+        }
+        return false;
+    }
+    
+    @Override
+    public void next() throws HyracksDataException {
+        foundNext = false;
+    }
+
+    @Override
+    public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
+        super.open(initialState, searchPred);
+        searchNextCursor();
+    }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySortedCursor.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySortedCursor.java
new file mode 100644
index 0000000..54a3ae8
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySortedCursor.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class LSMBTreeWithBuddySortedCursor extends
+		LSMBTreeWithBuddyAbstractCursor {
+	// TODO: This class can be removed and instead use a search cursor that uses
+	// a logic similar
+	// to the one in LSMRTreeWithAntiMatterTuplesSearchCursor
+	private boolean[] depletedBtreeCursors;
+	private int foundIn = -1;
+	private PermutingTupleReference buddyBtreeTuple;
+
+	public LSMBTreeWithBuddySortedCursor(ILSMIndexOperationContext opCtx,
+			int[] buddyBTreeFields) throws HyracksDataException {
+		super(opCtx);
+		this.buddyBtreeTuple = new PermutingTupleReference(buddyBTreeFields);
+		reset();
+	}
+
+	public ILSMIndexOperationContext getOpCtx() {
+		return opCtx;
+	}
+
+	@Override
+	public void reset() throws HyracksDataException {
+		depletedBtreeCursors = new boolean[numberOfTrees];
+		foundNext = false;
+		try {
+			for (int i = 0; i < numberOfTrees; i++) {
+				btreeCursors[i].reset();
+				try {
+					btreeAccessors[i].search(btreeCursors[i],
+							btreeRangePredicate);
+				} catch (IndexException e) {
+					throw new HyracksDataException(e);
+				}
+				if (btreeCursors[i].hasNext()) {
+					btreeCursors[i].next();
+				} else {
+					depletedBtreeCursors[i] = true;
+				}
+			}
+		} catch (IndexException e) {
+			e.printStackTrace();
+			throw new HyracksDataException(
+					"error while reseting the btrees of the lsm btree with buddy btree",
+					e);
+		} finally {
+			if (open) {
+				lsmHarness.endSearch(opCtx);
+			}
+		}
+	}
+
+	@Override
+	public boolean hasNext() throws HyracksDataException, IndexException {
+		while (!foundNext) {
+			frameTuple = null;
+
+			if (foundIn != -1) {
+				if (btreeCursors[foundIn].hasNext()) {
+					btreeCursors[foundIn].next();
+				} else {
+					depletedBtreeCursors[foundIn] = true;
+				}
+			}
+
+			foundIn = -1;
+			for (int i = 0; i < numberOfTrees; i++) {
+				if (depletedBtreeCursors[i])
+					continue;
+
+				if (frameTuple == null) {
+					frameTuple = btreeCursors[i].getTuple();
+					foundIn = i;
+					continue;
+				}
+
+				if (btreeCmp.compare(frameTuple, btreeCursors[i].getTuple()) > 0) {
+					frameTuple = btreeCursors[i].getTuple();
+					foundIn = i;
+				}
+			}
+
+			if (foundIn == -1)
+				return false;
+
+			boolean killed = false;
+			buddyBtreeTuple.reset(frameTuple);
+			for (int i = 0; i < foundIn; i++) {
+				try {
+					buddyBtreeCursors[i].reset();
+					buddyBtreeRangePredicate.setHighKey(buddyBtreeTuple, true);
+					btreeRangePredicate.setLowKey(buddyBtreeTuple, true);
+					btreeAccessors[i].search(btreeCursors[i],
+							btreeRangePredicate);
+				} catch (IndexException e) {
+					throw new HyracksDataException(e);
+				}
+				try {
+					if (btreeCursors[i].hasNext()) {
+						killed = true;
+						break;
+					}
+				} finally {
+					btreeCursors[i].close();
+				}
+			}
+			if (!killed) {
+				foundNext = true;
+			}
+		}
+
+		return true;
+	}
+
+	@Override
+	public void next() throws HyracksDataException {
+		foundNext = false;
+	}
+
+	@Override
+	public void open(ICursorInitialState initialState,
+			ISearchPredicate searchPred) throws HyracksDataException,
+			IndexException {
+		super.open(initialState, searchPred);
+
+		depletedBtreeCursors = new boolean[numberOfTrees];
+		foundNext = false;
+		for (int i = 0; i < numberOfTrees; i++) {
+			btreeCursors[i].reset();
+			try {
+				btreeAccessors[i].search(btreeCursors[i], btreeRangePredicate);
+			} catch (IndexException e) {
+				throw new HyracksDataException(e);
+			}
+			if (btreeCursors[i].hasNext()) {
+				btreeCursors[i].next();
+			} else {
+				depletedBtreeCursors[i] = true;
+			}
+		}
+	}
+
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java
new file mode 100644
index 0000000..c650de2
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
+
+public class LSMBuddyBTreeMergeCursor extends LSMIndexSearchCursor {
+
+	public LSMBuddyBTreeMergeCursor(ILSMIndexOperationContext opCtx) {
+		super(opCtx, true);
+	}
+
+	@Override
+	protected boolean isDeleted(PriorityQueueElement checkElement)
+			throws HyracksDataException, IndexException {
+		return false;
+	}
+
+	@Override
+	public void open(ICursorInitialState initialState,
+			ISearchPredicate searchPred) throws IndexException,
+			HyracksDataException {
+		LSMBTreeWithBuddyCursorInitialState lsmInitialState = (LSMBTreeWithBuddyCursorInitialState) initialState;
+		cmp = lsmInitialState.getBuddyBTreeCmp();
+		operationalComponents = lsmInitialState.getOperationalComponents();
+		// We intentionally set the lsmHarness to null so that we don't call
+		// lsmHarness.endSearch() because we already do that when we merge
+		// actual index.
+		lsmHarness = null;
+		int numBTrees = operationalComponents.size();
+		rangeCursors = new IIndexCursor[numBTrees];
+
+		RangePredicate btreePredicate = new RangePredicate(null, null, true,
+				true, cmp, cmp);
+		IIndexAccessor[] btreeAccessors = new ITreeIndexAccessor[numBTrees];
+		for (int i = 0; i < numBTrees; i++) {
+			ILSMComponent component = operationalComponents.get(i);
+			IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState
+					.getBuddyBTreeLeafFrameFactory().createFrame();
+			rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
+			BTree buddyBtree = (BTree) ((LSMBTreeWithBuddyDiskComponent) component)
+					.getBuddyBTree();
+			btreeAccessors[i] = buddyBtree.createAccessor(
+					NoOpOperationCallback.INSTANCE,
+					NoOpOperationCallback.INSTANCE);
+			btreeAccessors[i].search(rangeCursors[i], btreePredicate);
+		}
+		setPriorityQueueComparator();
+		initPriorityQueue();
+	}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
new file mode 100644
index 0000000..7582a33
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.btree.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+
+/**
+ * This is a TupleWriterFactory that can be used to reference the last writer it created
+ * and switch between actual and antimatter writer modes
+ * @author alamouda
+ *
+ */
+public class LSMBTreeRefrencingTupleWriterFactory extends TypeAwareTupleWriterFactory{
+	private static final long serialVersionUID = 1L;
+	private final ITypeTraits[] typeTraits;
+	private final int numKeyFields;
+	private boolean isDelete;
+	private LSMBTreeTupleWriter createdTupleWriter;
+
+	public LSMBTreeRefrencingTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) {
+		super(typeTraits);
+		this.typeTraits = typeTraits;
+		this.numKeyFields = numKeyFields;
+		this.isDelete = isDelete;
+	}
+
+	@Override
+	public ITreeIndexTupleWriter createTupleWriter() {
+		createdTupleWriter = new LSMBTreeTupleWriter(typeTraits, numKeyFields, isDelete);
+		return createdTupleWriter;
+	}
+
+	public void setMode(IndexOperation op){
+		if(op == IndexOperation.INSERT){
+			this.isDelete = false;
+			if(createdTupleWriter != null){
+				this.createdTupleWriter.setAntimatter(false);
+			}
+		} else if(op == IndexOperation.DELETE){
+			this.isDelete = true;
+			if(createdTupleWriter != null){
+				this.createdTupleWriter.setAntimatter(true);
+			}
+		}
+	}
+
+	public LSMBTreeTupleWriter getCreatedTupleWriter() {
+		return createdTupleWriter;
+	}
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
index 9426282..2206b1e 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
@@ -21,7 +21,7 @@
 import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
 
 public class LSMBTreeTupleWriter extends TypeAwareTupleWriter {
-    private final boolean isAntimatter;
+    private boolean isAntimatter;
     private final int numKeyFields;
 
     public LSMBTreeTupleWriter(ITypeTraits[] typeTraits, int numKeyFields, boolean isAntimatter) {
@@ -78,4 +78,9 @@
         // Set leftmost bit to 1.
         targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
     }
+
+    // Allow using the same writer for both delete and insert tuples
+    public void setAntimatter(boolean isAntimatter) {
+        this.isAntimatter = isAntimatter;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
index 0cbefc9..f184f96 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
@@ -22,13 +22,11 @@
 public class LSMBTreeTupleWriterFactory extends TypeAwareTupleWriterFactory {
 
 	private static final long serialVersionUID = 1L;
-	private final ITypeTraits[] typeTraits;
 	private final int numKeyFields;
 	private final boolean isDelete;
 	
 	public LSMBTreeTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) {
 		super(typeTraits);
-		this.typeTraits = typeTraits;
 		this.numKeyFields = numKeyFields;
 		this.isDelete = isDelete;
 	}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 88148c3..6d7564b 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -29,9 +29,14 @@
 import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
 import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
 import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.ExternalBTree;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy;
 import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTree;
 import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager;
 import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -81,4 +86,112 @@
                 opTracker, ioScheduler, ioOpCallback, needKeyDupCheck);
         return lsmTree;
     }
+
+    public static ExternalBTree createExternalBTree(FileReference file, IBufferCache diskBufferCache,
+            IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+            int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            int startWithVersion) {
+        LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+                cmpFactories.length, false);
+        LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+                cmpFactories.length, true);
+        LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits,
+                cmpFactories.length);
+        ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
+        ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
+        ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
+        ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
+        ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+        IFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+                metaFrameFactory);
+        // This is the tuple writer that can do both inserts and deletes
+        LSMBTreeRefrencingTupleWriterFactory referencingTupleWriterFactory = new LSMBTreeRefrencingTupleWriterFactory(
+                typeTraits, cmpFactories.length, false);
+        // This is the leaf frame factory for transaction components since it
+        // can be used for both inserts and deletes
+        ITreeIndexFrameFactory dualLeafFrameFactory = new BTreeNSMLeafFrameFactory(referencingTupleWriterFactory);
+
+        TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
+                typeTraits.length);
+        TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
+
+        BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
+                bloomFilterKeyFields);
+
+        // This is the component factory for transactions
+        TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, interiorFrameFactory, dualLeafFrameFactory, cmpFactories, typeTraits.length);
+
+        ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(diskFileMapProvider, file, diskBTreeFactory);
+
+        // the disk only index uses an empty ArrayList for virtual buffer caches
+        ExternalBTree lsmTree = new ExternalBTree(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
+                fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
+                bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
+                opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, startWithVersion);
+        return lsmTree;
+    }
+
+    public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(FileReference file, IBufferCache diskBufferCache,
+            IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields,
+            int startWithVersion) {
+
+        ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
+        IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length];
+        for (int i = 0; i < buddyBtreeTypeTraits.length; i++) {
+            buddyBtreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
+            buddyBtreeCmpFactories[i] = cmpFactories[buddyBTreeFields[i]];
+        }
+
+        TypeAwareTupleWriterFactory buddyBtreeTupleWriterFactory = new TypeAwareTupleWriterFactory(buddyBtreeTypeTraits);
+        ITreeIndexFrameFactory buddyBtreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+                buddyBtreeTupleWriterFactory);
+        ITreeIndexFrameFactory buddyBtreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(buddyBtreeTupleWriterFactory);
+
+        LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+                cmpFactories.length, false);
+        LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits,
+                cmpFactories.length);
+        ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
+        ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
+        ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
+        ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+        IFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+                metaFrameFactory);
+
+        TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
+                typeTraits.length);
+
+        TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
+
+        int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length];
+        for (int i = 0; i < buddyBtreeCmpFactories.length; i++) {
+            bloomFilterKeyFields[i] = i;
+        }
+        BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
+                bloomFilterKeyFields);
+
+        // buddy b-tree factory
+        TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory,
+                buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
+
+        ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(diskFileMapProvider, file,
+                diskBTreeFactory, diskBuddyBTreeFactory);
+
+        // the disk only index uses an empty ArrayList for virtual buffer caches
+        ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(interiorFrameFactory, insertLeafFrameFactory,
+                buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, bulkLoadBTreeFactory, diskBTreeFactory,
+                diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider, bloomFilterFalsePositiveRate,
+                mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, buddyBtreeCmpFactories,
+                buddyBTreeFields, startWithVersion);
+        return lsmTree;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index 6d460d6..ba3d9e3 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -15,6 +15,7 @@
 
 package edu.uci.ics.hyracks.storage.am.lsm.common.api;
 
+import java.io.IOException;
 import java.util.Comparator;
 import java.util.List;
 
@@ -46,4 +47,31 @@
     public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException;
 
     public Comparator<String> getFileNameComparator();
+
+    /**
+     * @return delete existing transaction disk component file reference
+     * @throws HyracksDataException
+     */
+    public void deleteTransactionFiles() throws HyracksDataException;
+
+    /**
+     * Rename files of a transaction removing the transaction prefix and return the component file reference in order to be committed
+     * 
+     * @return the renamed component file references
+     * @throws HyracksDataException
+     */
+    public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException;
+
+    /**
+     * Recover transaction files without returning them
+     * 
+     * @throws HyracksDataException
+     */
+    public void recoverTransaction() throws HyracksDataException;
+
+    /**
+     * @return a reference to the transaction disk component file reference
+     * @throws IOException
+     */
+    public LSMComponentFileReferences getNewTransactionFileReference() throws IOException;
 }
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
index 6020093..b8f590f 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
@@ -45,9 +45,9 @@
 
     public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
 
-    public void addComponent(ILSMComponent index);
+    public void addComponent(ILSMComponent index) throws HyracksDataException;
 
-    public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents);
+    public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents) throws HyracksDataException;
 
     public void changeMutableComponent();
 
@@ -64,5 +64,4 @@
     public void getOperationalComponents(ILSMIndexOperationContext ctx);
 
     public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException;
-
 }
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
new file mode 100644
index 0000000..cc4d65e
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+
+// An interface containing the new methods introduced for bulk transactions
+public interface ITwoPCIndex {
+    /**
+     * This function is used to create a BulkLoader for a transaction that is capable of insertions and deletions
+     * and the bulk loaded component is hidden from the index
+     */
+    public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex) throws TreeIndexException;
+    
+    /**
+     * This function is used to commit the previous transaction if it was resulted in creating any components
+     */
+    public void commitTransaction() throws TreeIndexException, HyracksDataException, IndexException;
+    
+    /**
+     * This function is used to abort the last transaction
+     */
+    public void abortTransaction() throws TreeIndexException;
+    
+    /**
+     * This function is used to recover a transaction if the system crashed after the decision to commit
+     */
+    public void recoverTransaction() throws TreeIndexException;
+    
+    /**
+     * This function is used to add the committed disk component to the appropriate list and reflect the changes
+     */
+    public void commitTransactionDiskComponent(ILSMComponent newComponent) throws IndexException, HyracksDataException;
+    
+    /**
+     * This function is used to create a version specific accessor to search a specific version
+     */
+    public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion) throws HyracksDataException;
+    
+    /**
+     * This function is used to get the first components list
+     */
+    public List<ILSMComponent> getFirstComponentList();
+    
+    /**
+     * This function is used to get teh second components list
+     */
+    public List<ILSMComponent> getSecondComponentList();
+    
+    /**
+     * This function is used to get the current version id of the index
+     */
+    public int getCurrentVersion();
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
new file mode 100644
index 0000000..122f63e
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import edu.uci.ics.hyracks.api.io.FileReference;
+
+public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager {
+    public AbortRecoverLSMIndexFileManager(FileReference file) {
+        super(null, file, null);
+    }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 3d7512f..e5fcf0b 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -27,7 +27,6 @@
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
@@ -84,6 +83,26 @@
         }
     }
 
+    // The constructor used by external indexes
+    public AbstractLSMIndex(IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
+            IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback) {
+        this.diskBufferCache = diskBufferCache;
+        this.diskFileMapProvider = diskFileMapProvider;
+        this.fileManager = fileManager;
+        this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
+        this.ioScheduler = ioScheduler;
+        this.ioOpCallback = ioOpCallback;
+        lsmHarness = new ExternalIndexHarness(this, mergePolicy, opTracker);
+        isActivated = false;
+        diskComponents = new LinkedList<ILSMComponent>();
+        // Memory related objects are nulled
+        this.virtualBufferCaches = null;
+        memoryComponents = null;
+        currentMutableComponentId = null;
+        flushRequests = null;
+    }
+
     protected void forceFlushDirtyPages(ITreeIndex treeIndex) throws HyracksDataException {
         int fileId = treeIndex.getFileId();
         IBufferCache bufferCache = treeIndex.getBufferCache();
@@ -147,12 +166,13 @@
     }
 
     @Override
-    public void addComponent(ILSMComponent c) {
+    public void addComponent(ILSMComponent c) throws HyracksDataException {
         diskComponents.add(0, c);
     }
 
     @Override
-    public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents) {
+    public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+            throws HyracksDataException {
         int swapIndex = diskComponents.indexOf(mergedComponents.get(0));
         diskComponents.removeAll(mergedComponents);
         diskComponents.add(swapIndex, newComponent);
@@ -215,4 +235,9 @@
     public String toString() {
         return "LSMIndex [" + fileManager.getBaseDir() + "]";
     }
+    
+    @Override
+    public boolean hasMemoryComponents() {
+        return true;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index a6425ce..1676560 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -17,6 +17,9 @@
 
 import java.io.File;
 import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.text.Format;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -41,6 +44,7 @@
 
     protected static final String SPLIT_STRING = "_";
     protected static final String BLOOM_FILTER_STRING = "f";
+    protected static final String TRANSACTION_PREFIX = ".T";
 
     protected final IFileMapProvider fileMapProvider;
 
@@ -256,6 +260,22 @@
         return baseDir;
     }
 
+    public void recoverTransaction() throws HyracksDataException {
+        File dir = new File(baseDir);
+        String[] files = dir.list(transactionFileNameFilter);
+        try {
+            if (files.length == 0) {
+                // Do nothing
+            } else if (files.length > 1) {
+                throw new HyracksDataException("Found more than one transaction");
+            } else {
+                Files.delete(Paths.get(dir.getPath() + File.separator + files[0]));
+            }
+        } catch (IOException e) {
+            throw new HyracksDataException("Failed to recover transaction", e);
+        }
+    }
+
     protected class ComparableFileName implements Comparable<ComparableFileName> {
         public final FileReference fileRef;
         public final String fullPath;
@@ -291,4 +311,90 @@
             return -a.interval[1].compareTo(b.interval[1]);
         }
     }
+
+    // This function is used to delete transaction files for aborted transactions
+    @Override
+    public void deleteTransactionFiles() throws HyracksDataException {
+        File dir = new File(baseDir);
+        String[] files = dir.list(transactionFileNameFilter);
+        if (files.length == 0) {
+            // Do nothing
+        } else if (files.length > 1) {
+            throw new HyracksDataException("Found more than one transaction");
+        } else {
+            //create transaction filter
+            FilenameFilter transactionFilter = createTransactionFilter(files[0], true);
+            String[] componentsFiles = dir.list(transactionFilter);
+            for (String fileName : componentsFiles) {
+                try {
+                    String absFileName = dir.getPath() + File.separator + fileName;
+                    Files.delete(Paths.get(absFileName));
+                } catch (IOException e) {
+                    throw new HyracksDataException("Failed to delete transaction files", e);
+                }
+            }
+            // delete the txn lock file
+            String absFileName = dir.getPath() + File.separator + files[0];
+            try {
+                Files.delete(Paths.get(absFileName));
+            } catch (IOException e) {
+                throw new HyracksDataException("Failed to delete transaction files", e);
+            }
+        }
+    }
+
+    @Override
+    public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
+        return null;
+    }
+
+    @Override
+    public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
+        return null;
+    }
+
+    protected static FilenameFilter transactionFileNameFilter = new FilenameFilter() {
+        public boolean accept(File dir, String name) {
+            return name.startsWith(".T");
+        }
+    };
+
+    protected static FilenameFilter dummyFilter = new FilenameFilter() {
+        public boolean accept(File dir, String name) {
+            return true;
+        }
+    };
+
+    protected static FilenameFilter createTransactionFilter(String transactionFileName, final boolean inclusive) {
+        final String timeStamp = transactionFileName.substring(transactionFileName.indexOf(TRANSACTION_PREFIX)
+                + TRANSACTION_PREFIX.length());
+        return new FilenameFilter() {
+            public boolean accept(File dir, String name) {
+                if (inclusive) {
+                    return name.startsWith(timeStamp);
+                } else {
+                    return !name.startsWith(timeStamp);
+                }
+            }
+        };
+    }
+
+    protected FilenameFilter getTransactionFileFilter(boolean inclusive) {
+        File dir = new File(baseDir);
+        String[] files = dir.list(transactionFileNameFilter);
+        if (files.length == 0) {
+            return dummyFilter;
+        } else {
+            return createTransactionFilter(files[0], inclusive);
+        }
+    }
+
+    protected FilenameFilter getCompoundFilter(final FilenameFilter filter1, final FilenameFilter filter2) {
+        return new FilenameFilter() {
+            @Override
+            public boolean accept(File dir, String name) {
+                return (filter1.accept(dir, name) && filter2.accept(dir, name));
+            }
+        };
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java
new file mode 100644
index 0000000..8c763cb
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java
@@ -0,0 +1,338 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
+
+public class ExternalIndexHarness extends LSMHarness {
+    private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName());
+
+
+    public ExternalIndexHarness(ILSMIndexInternal lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker) {
+        super(lsmIndex,mergePolicy,opTracker);
+    }
+
+    @Override
+    protected boolean getAndEnterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean isTryOperation)
+            throws HyracksDataException {
+        synchronized (opTracker) {
+            while (true) {
+                lsmIndex.getOperationalComponents(ctx);
+                // Before entering the components, prune those corner cases that indeed should not proceed.
+                switch (opType) {
+                    case MERGE:
+                        if (ctx.getComponentHolder().size() < 2) {
+                            // There is only a single component. There is nothing to merge.
+                            return false;
+                        }
+                    default:
+                        break;
+                }
+                if (enterComponents(ctx, opType)) {
+                    return true;
+                } else if (isTryOperation) {
+                    return false;
+                }
+            }
+        }
+    }
+
+    @Override
+    protected boolean enterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType) throws HyracksDataException {
+        List<ILSMComponent> components = ctx.getComponentHolder();
+        int numEntered = 0;
+        boolean entranceSuccessful = false;
+        try {
+            for (ILSMComponent c : components) {
+                if (!c.threadEnter(opType, false)) {
+                    break;
+                }
+                numEntered++;
+            }
+            entranceSuccessful = numEntered == components.size();
+        } finally {
+            if (!entranceSuccessful) {
+                for (ILSMComponent c : components) {
+                    if (numEntered == 0) {
+                        break;
+                    }
+                    c.threadExit(opType, true, false);
+                    numEntered--;
+                }
+                return false;
+            }
+        }
+        // Check if there is any action that is needed to be taken based on the operation type
+        switch (opType) {
+            case MERGE:
+                lsmIndex.getIOOperationCallback().beforeOperation(LSMOperationType.MERGE);
+            default:
+                break;
+        }
+        opTracker.beforeOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
+        return true;
+    }
+
+    private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMComponent newComponent,
+            boolean failedOperation) throws HyracksDataException, IndexException {
+        synchronized (opTracker) {
+            try {
+                // First check if there is any action that is needed to be taken based on the state of each component.
+                for (ILSMComponent c : ctx.getComponentHolder()) {
+                    c.threadExit(opType, failedOperation, false);
+                    switch (c.getState()) {
+                        case INACTIVE:
+                            ((AbstractDiskLSMComponent) c).destroy();
+                            break;
+                        default:
+                            break;
+                    }
+                }
+                // Then, perform any action that is needed to be taken based on the operation type.
+                switch (opType) {
+                    case MERGE:
+                        // newComponent is null if the merge op. was not performed.
+                        if (newComponent != null) {
+                            beforeSubsumeMergedComponents(newComponent, ctx.getComponentHolder());
+                            lsmIndex.subsumeMergedComponents(newComponent, ctx.getComponentHolder());
+                            mergePolicy.diskComponentAdded(lsmIndex, fullMergeIsRequested.get());
+                        }
+                        break;
+                    default:
+                        break;
+                }
+            } finally {
+                opTracker.afterOperation(lsmIndex, opType, ctx.getSearchOperationCallback(),
+                        ctx.getModificationCallback());
+            }
+        }
+    }
+
+    @Override
+    public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException,
+            IndexException {
+        throw new IndexException("2PC LSM Inedx doesn't support modify");
+    }
+
+    @Override
+    public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
+            throws HyracksDataException, IndexException {
+        throw new IndexException("2PC LSM Inedx doesn't support modify");
+    }
+
+    @Override
+    public void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred)
+            throws HyracksDataException, IndexException {
+        LSMOperationType opType = LSMOperationType.SEARCH;
+        getAndEnterComponents(ctx, opType, false);
+        try {
+            lsmIndex.search(ctx, cursor, pred);
+        } catch (HyracksDataException | IndexException e) {
+            exitComponents(ctx, opType, null, true);
+            throw e;
+        }
+    }
+
+    @Override
+    public void endSearch(ILSMIndexOperationContext ctx) throws HyracksDataException {
+        if (ctx.getOperation() == IndexOperation.SEARCH) {
+            try {
+                exitComponents(ctx, LSMOperationType.SEARCH, null, false);
+            } catch (IndexException e) {
+                throw new HyracksDataException(e);
+            }
+        }
+    }
+
+    @Override
+    public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException, IndexException {
+        if (!getAndEnterComponents(ctx, LSMOperationType.MERGE, true)) {
+            callback.afterFinalize(LSMOperationType.MERGE, null);
+            return;
+        }
+        lsmIndex.scheduleMerge(ctx, callback);
+    }
+
+    @Override
+    public void scheduleFullMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException, IndexException {
+        fullMergeIsRequested.set(true);
+        if (!getAndEnterComponents(ctx, LSMOperationType.MERGE, true)) {
+            // If the merge cannot be scheduled because there is already an ongoing merge on subset/all of the components, then
+            // whenever the current merge has finished, it will schedule the full merge again.
+            callback.afterFinalize(LSMOperationType.MERGE, null);
+            return;
+        }
+        fullMergeIsRequested.set(false);
+        lsmIndex.scheduleMerge(ctx, callback);
+    }
+
+    @Override
+    public void merge(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException,
+            IndexException {
+        if (LOGGER.isLoggable(Level.INFO)) {
+            LOGGER.info("Started a merge operation for index: " + lsmIndex + " ...");
+        }
+
+        ILSMComponent newComponent = null;
+        try {
+            newComponent = lsmIndex.merge(operation);
+            operation.getCallback().afterOperation(LSMOperationType.MERGE, ctx.getComponentHolder(), newComponent);
+            lsmIndex.markAsValid(newComponent);
+        } finally {
+            exitComponents(ctx, LSMOperationType.MERGE, newComponent, false);
+            operation.getCallback().afterFinalize(LSMOperationType.MERGE, newComponent);
+        }
+        if (LOGGER.isLoggable(Level.INFO)) {
+            LOGGER.info("Finished the merge operation for index: " + lsmIndex);
+        }
+    }
+
+    @Override
+    public void addBulkLoadedComponent(ILSMComponent c) throws HyracksDataException, IndexException {
+        lsmIndex.markAsValid(c);
+        lsmIndex.addComponent(c);
+        // Enter the component
+        enterComponent(c);
+        mergePolicy.diskComponentAdded(lsmIndex, false);
+    }
+
+    // Three differences from  addBulkLoadedComponent
+    // 1. this needs synchronization since others might be accessing the index (specifically merge operations that might change the lists of components)
+    // 2. the actions taken by the index itself are different
+    // 3. the component has already been marked valid by the bulk update operation
+    public void addTransactionComponents(ILSMComponent newComponent) throws HyracksDataException, IndexException {
+        ITwoPCIndex index = (ITwoPCIndex) lsmIndex;
+        synchronized (opTracker) {
+            List<ILSMComponent> newerList;
+            List<ILSMComponent> olderList;
+            if (index.getCurrentVersion() == 0) {
+                newerList = index.getFirstComponentList();
+                olderList = index.getSecondComponentList();
+            } else {
+                newerList = index.getSecondComponentList();
+                olderList = index.getFirstComponentList();
+            }
+            // Exit components in old version of the index so they are ready to be
+            // deleted if they are not needed anymore
+            for (ILSMComponent c : olderList) {
+                exitComponent(c);
+            }
+            // Enter components in the newer list
+            for (ILSMComponent c : newerList) {
+                enterComponent(c);
+            }
+            if (newComponent != null) {
+                // Enter new component
+                enterComponent(newComponent);
+            }
+            index.commitTransactionDiskComponent(newComponent);
+            mergePolicy.diskComponentAdded(lsmIndex, fullMergeIsRequested.get());
+        }
+    }
+
+    @Override
+    public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException {
+        callback.afterFinalize(LSMOperationType.FLUSH, null);
+    }
+
+    @Override
+    public void flush(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException,
+            IndexException {
+    }
+
+    @Override
+    public ILSMOperationTracker getOperationTracker() {
+        return opTracker;
+    }
+
+    public void beforeSubsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+            throws HyracksDataException {
+        ITwoPCIndex index = (ITwoPCIndex) lsmIndex;
+        // check if merge will affect the first list
+        if (index.getFirstComponentList().containsAll(mergedComponents)) {
+            // exit un-needed components
+            for (ILSMComponent c : mergedComponents) {
+                exitComponent(c);
+            }
+            // enter new component
+            enterComponent(newComponent);
+        }
+        // check if merge will affect the second list
+        if (index.getSecondComponentList().containsAll(mergedComponents)) {
+            // exit un-needed components
+            for (ILSMComponent c : mergedComponents) {
+                exitComponent(c);
+            }
+            // enter new component
+            enterComponent(newComponent);
+        }
+    }
+
+    // The two methods: enterComponent and exitComponent are used to control
+    // when components are to be deleted from disk
+    private void enterComponent(ILSMComponent diskComponent) throws HyracksDataException {
+        diskComponent.threadEnter(LSMOperationType.SEARCH, false);
+    }
+
+    private void exitComponent(ILSMComponent diskComponent) throws HyracksDataException {
+        diskComponent.threadExit(LSMOperationType.SEARCH, false, false);
+        if (diskComponent.getState() == ILSMComponent.ComponentState.INACTIVE) {
+            ((AbstractDiskLSMComponent) diskComponent).destroy();
+        }
+    }
+
+    public void indexFirstTimeActivated() throws HyracksDataException {
+        ITwoPCIndex index = (ITwoPCIndex) lsmIndex;
+        // Enter disk components <-- To avoid deleting them when they are
+        // still needed-->
+        for (ILSMComponent c : index.getFirstComponentList()) {
+            enterComponent(c);
+        }
+        for (ILSMComponent c : index.getSecondComponentList()) {
+            enterComponent(c);
+        }
+    }
+
+    public void indexClear() throws HyracksDataException {
+        ITwoPCIndex index = (ITwoPCIndex) lsmIndex;
+        for (ILSMComponent c : index.getFirstComponentList()) {
+            exitComponent(c);
+        }
+        for (ILSMComponent c : index.getSecondComponentList()) {
+            exitComponent(c);
+        }
+    }
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 8420c73..a88b982 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -39,10 +39,10 @@
 public class LSMHarness implements ILSMHarness {
     private static final Logger LOGGER = Logger.getLogger(LSMHarness.class.getName());
 
-    private final ILSMIndexInternal lsmIndex;
-    private final ILSMMergePolicy mergePolicy;
-    private final ILSMOperationTracker opTracker;
-    private final AtomicBoolean fullMergeIsRequested;
+    protected final ILSMIndexInternal lsmIndex;
+    protected final ILSMMergePolicy mergePolicy;
+    protected final ILSMOperationTracker opTracker;
+    protected final AtomicBoolean fullMergeIsRequested;
 
     public LSMHarness(ILSMIndexInternal lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker) {
         this.lsmIndex = lsmIndex;
@@ -51,7 +51,7 @@
         fullMergeIsRequested = new AtomicBoolean();
     }
 
-    private boolean getAndEnterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean isTryOperation)
+    protected boolean getAndEnterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean isTryOperation)
             throws HyracksDataException {
         synchronized (opTracker) {
             while (true) {
@@ -90,7 +90,7 @@
         }
     }
 
-    private boolean enterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType) throws HyracksDataException {
+    protected boolean enterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType) throws HyracksDataException {
         List<ILSMComponent> components = ctx.getComponentHolder();
         int numEntered = 0;
         boolean entranceSuccessful = false;
@@ -217,7 +217,7 @@
         return modify(ctx, tryOperation, tuple, opType);
     }
 
-    private boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple,
+    protected boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple,
             LSMOperationType opType) throws HyracksDataException, IndexException {
         if (!getAndEnterComponents(ctx, opType, tryOperation)) {
             return false;
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index ff0d9e3..44df9af 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -822,4 +822,9 @@
     public String toString() {
         return "LSMInvertedIndex [" + fileManager.getBaseDir() + "]";
     }
+    
+    @Override
+    public boolean hasMemoryComponents() {
+        return true;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
index c70c2d5..f4a3916 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
@@ -202,4 +202,9 @@
     public IBinaryComparatorFactory[] getTokenCmpFactories() {
         return tokenCmpFactories;
     }
+
+    @Override
+    public boolean hasMemoryComponents() {
+        return true;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 6aa3253..1684cf0 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -673,4 +673,9 @@
     public IBinaryComparatorFactory[] getTokenCmpFactories() {
         return tokenCmpFactories;
     }
+    
+    @Override
+    public boolean hasMemoryComponents() {
+        return true;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
new file mode 100644
index 0000000..7b823b3
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public class ExternalRTreeDataflowHelper extends LSMRTreeDataflowHelper {
+
+    private int version;
+
+    public ExternalRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+            IBinaryComparatorFactory[] btreeComparatorFactories,
+            IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
+            ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+            ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version) {
+        super(opDesc, ctx, partition, null, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
+                mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, btreeFields);
+        this.version = version;
+    }
+
+    public ExternalRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+            double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] btreeComparatorFactories,
+            IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
+            ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+            ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version) {
+        super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, btreeComparatorFactories,
+                valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
+                ioOpCallbackFactory, linearizeCmpFactory, btreeFields);
+        this.version = version;
+    }
+
+    @Override
+    public IIndex getIndexInstance() {
+        if (index != null)
+            return index;
+        synchronized (lcManager) {
+            long resourceID;
+            try {
+                resourceID = getResourceID();
+            } catch (HyracksDataException e) {
+                return null;
+            }
+            try {
+                index = lcManager.getIndex(resourceID);
+            } catch (HyracksDataException e) {
+                return null;
+            }
+        }
+        return index;
+    }
+
+    @Override
+    protected ITreeIndex createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
+            IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+            IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+            ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
+            RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory)
+            throws HyracksDataException {
+        try {
+            return LSMRTreeUtils.createExternalRTree(file, diskBufferCache, diskFileMapProvider, typeTraits,
+                    rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+                    bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
+                    ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, btreeFields, version);
+        } catch (TreeIndexException e) {
+            throw new HyracksDataException(e);
+        }
+    }
+
+    public int getTargetVersion() {
+        return version;
+    }
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
new file mode 100644
index 0000000..81b2582
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
+
+public class ExternalRTreeDataflowHelperFactory extends LSMRTreeDataflowHelperFactory {
+
+    private static final long serialVersionUID = 1L;
+    private int version;
+
+    public ExternalRTreeDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories,
+            RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories,
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+            ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+            ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILinearizeComparatorFactory linearizeCmpFactory,
+            double bloomFilterFalsePositiveRate, int[] btreeFields, int version) {
+        super(valueProviderFactories, rtreePolicyType, btreeComparatorFactories, null, mergePolicyFactory,
+                mergePolicyProperties, opTrackerFactory, ioSchedulerProvider, ioOpCallbackFactory, linearizeCmpFactory,
+                bloomFilterFalsePositiveRate, btreeFields);
+        this.version = version;
+    }
+
+    @Override
+    public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+            int partition) {
+        return new ExternalRTreeDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
+                btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
+                mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
+                ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, linearizeCmpFactory, btreeFields, version);
+    }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index 8d6c0da..f316503 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -39,7 +39,7 @@
 import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMRTreeDataflowHelper extends AbstractLSMRTreeDataflowHelper {
-    private int[] btreeFields;
+    protected int[] btreeFields;
 
     public LSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
             List<IVirtualBufferCache> virtualBufferCaches, IBinaryComparatorFactory[] btreeComparatorFactories,
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index beb6c20..7548398 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -35,11 +35,11 @@
 
     private static final long serialVersionUID = 1L;
 
-    private final IBinaryComparatorFactory[] btreeComparatorFactories;
-    private final IPrimitiveValueProviderFactory[] valueProviderFactories;
-    private final RTreePolicyType rtreePolicyType;
-    private final ILinearizeComparatorFactory linearizeCmpFactory;
-    private final int[] btreeFields;
+    protected final IBinaryComparatorFactory[] btreeComparatorFactories;
+    protected final IPrimitiveValueProviderFactory[] valueProviderFactories;
+    protected final RTreePolicyType rtreePolicyType;
+    protected final ILinearizeComparatorFactory linearizeCmpFactory;
+    protected final int[] btreeFields;
 
     public LSMRTreeDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories,
             RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories,
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 2b9714d..a69f1f8 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -65,8 +65,8 @@
     // For creating RTree's used in flush and merge.
     protected final ILSMComponentFactory componentFactory;
 
-    private IBinaryComparatorFactory[] btreeCmpFactories;
-    private IBinaryComparatorFactory[] rtreeCmpFactories;
+    protected IBinaryComparatorFactory[] btreeCmpFactories;
+    protected IBinaryComparatorFactory[] rtreeCmpFactories;
 
     // Common for in-memory and on-disk components.
     protected final ITreeIndexFrameFactory rtreeInteriorFrameFactory;
@@ -114,6 +114,31 @@
         this.comparatorFields = comparatorFields;
         this.linearizerArray = linearizerArray;
     }
+    
+    /*
+     * For External indexes with no memory components
+     */
+    public AbstractLSMRTree(ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+            ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
+            ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileManager,
+            ILSMComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, int fieldCount,
+            IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+            ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
+            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback) {
+        super(componentFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate,
+                mergePolicy, opTracker, ioScheduler, ioOpCallback);
+        this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
+        this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
+        this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
+        this.btreeLeafFrameFactory = btreeLeafFrameFactory;
+        this.componentFactory = componentFactory;
+        this.btreeCmpFactories = btreeCmpFactories;
+        this.rtreeCmpFactories = rtreeCmpFactories;
+        this.linearizer = linearizer;
+        this.comparatorFields = comparatorFields;
+        this.linearizerArray = linearizerArray;
+    }
 
     @Override
     public synchronized void create() throws HyracksDataException {
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
new file mode 100644
index 0000000..d26d330
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -0,0 +1,758 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+/**
+ * This is an lsm r-tree that does not have memory component and is modified
+ * only by bulk loading and addition of disk components as of this point, it is
+ * intended for use with external dataset indexes only.
+ * 
+ * @author alamouda
+ */
+public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
+
+    // A second disk component list that will be used when a transaction is
+    // committed and will be seen by subsequent accessors
+    private List<ILSMComponent> secondDiskComponents;
+    // A pointer that points to the current most recent list (either
+    // diskComponents = 0, or secondDiskComponents = 1). It starts with -1 to
+    // indicate first time activation
+    private int version = -1;
+    private int fieldCount;
+
+    public ExternalRTree(ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+            ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
+            ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
+            TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
+            BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
+            IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+            IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+            int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            int[] buddyBTreeFields, int version) {
+        super(rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
+                fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate,
+                diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
+                linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields);
+        this.secondDiskComponents = new LinkedList<ILSMComponent>();
+        this.version = version;
+        this.fieldCount = fieldCount;
+    }
+
+    // This method is used to create a target for a bulk modify operation. This
+    // component must then eventually be either committed or deleted
+    private ILSMComponent createTransactionTarget() throws HyracksDataException, IndexException {
+        LSMComponentFileReferences componentFileRefs;
+        try {
+            componentFileRefs = fileManager.getNewTransactionFileReference();
+        } catch (IOException e) {
+            throw new HyracksDataException("Failed to create transaction components", e);
+        }
+        return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
+                componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
+    }
+
+    // The subsume merged components is overridden to account for:
+    // 1. the number of readers of components
+    // 2. maintaining two versions of the index
+    @Override
+    public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+            throws HyracksDataException {
+        // determine which list is the new one
+        List<ILSMComponent> newerList;
+        List<ILSMComponent> olderList;
+        if (version == 0) {
+            newerList = diskComponents;
+            olderList = secondDiskComponents;
+        } else {
+            newerList = secondDiskComponents;
+            olderList = diskComponents;
+        }
+        // check if merge will affect the older list
+        if (olderList.containsAll(mergedComponents)) {
+            int swapIndex = olderList.indexOf(mergedComponents.get(0));
+            olderList.removeAll(mergedComponents);
+            olderList.add(swapIndex, newComponent);
+        }
+        // The new list will always have all the merged components
+        int swapIndex = newerList.indexOf(mergedComponents.get(0));
+        newerList.removeAll(mergedComponents);
+        newerList.add(swapIndex, newComponent);
+    }
+
+    // This method is used by the merge policy when it needs to check if a merge
+    // is needed.
+    // It only needs to return the newer list
+    @Override
+    public List<ILSMComponent> getImmutableComponents() {
+        if (version == 0) {
+            return diskComponents;
+        } else {
+            return secondDiskComponents;
+        }
+    }
+
+    // This function should only be used when a transaction fail. it doesn't
+    // take any parameters since there can only be
+    // a single transaction and hence a single transaction component on disk
+    public void deleteTransactionComponent() throws HyracksDataException {
+        fileManager.deleteTransactionFiles();
+    }
+
+    // This function in an instance of this index is only used after a bulk load
+    // is successful
+    // it will therefore add the component to the first list and enter it.
+    @Override
+    public void addComponent(ILSMComponent c) throws HyracksDataException {
+        if (version == 0) {
+            diskComponents.add(0, c);
+        } else if (version == 1) {
+            secondDiskComponents.add(0, c);
+        }
+    }
+
+    // This function is used when a new component is to be committed.
+    public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
+
+        // determine which list is the new one and flip the pointer
+        List<ILSMComponent> newerList;
+        List<ILSMComponent> olderList;
+        if (version == 0) {
+            newerList = diskComponents;
+            olderList = secondDiskComponents;
+            version = 1;
+        } else {
+            newerList = secondDiskComponents;
+            olderList = diskComponents;
+            version = 0;
+        }
+        // Remove components from list
+        olderList.clear();
+        // Add components
+        olderList.addAll(newerList);
+        if (newComponent != null) {
+            // Add it to the list
+            olderList.add(0, newComponent);
+        }
+    }
+
+    @Override
+    public synchronized void activate() throws HyracksDataException {
+        if (isActivated) {
+            throw new HyracksDataException("Failed to activate the index since it is already activated.");
+        }
+
+        if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
+            //First time activation
+            List<LSMComponentFileReferences> validFileReferences;
+            try {
+                validFileReferences = fileManager.cleanupAndGetValidFiles();
+            } catch (IndexException e) {
+                throw new HyracksDataException(e);
+            }
+            for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
+                LSMRTreeDiskComponent component;
+                try {
+                    component = createDiskComponent(componentFactory,
+                            lsmComonentFileReference.getInsertIndexFileReference(),
+                            lsmComonentFileReference.getDeleteIndexFileReference(),
+                            lsmComonentFileReference.getBloomFilterFileReference(), false);
+                } catch (IndexException e) {
+                    throw new HyracksDataException(e);
+                }
+                diskComponents.add(component);
+                secondDiskComponents.add(component);
+            }
+            ((ExternalIndexHarness) lsmHarness).indexFirstTimeActivated();
+        } else {
+            // This index has been opened before or is brand new with no components
+            // components. It should also maintain the version pointer
+            for (ILSMComponent c : diskComponents) {
+                LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+                RTree rtree = component.getRTree();
+                BTree btree = component.getBTree();
+                BloomFilter bloomFilter = component.getBloomFilter();
+                rtree.activate();
+                btree.activate();
+                bloomFilter.activate();
+            }
+            for (ILSMComponent c : secondDiskComponents) {
+                // Only activate non shared components
+                if (!diskComponents.contains(c)) {
+                    LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+                    RTree rtree = component.getRTree();
+                    BTree btree = component.getBTree();
+                    BloomFilter bloomFilter = component.getBloomFilter();
+                    rtree.activate();
+                    btree.activate();
+                    bloomFilter.activate();
+                }
+            }
+        }
+        isActivated = true;
+    }
+
+    @Override
+    public synchronized void create() throws HyracksDataException {
+        super.create();
+        secondDiskComponents.clear();
+    }
+
+    // we override this method because this index uses a different opcontext
+    @Override
+    public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+            throws HyracksDataException, IndexException {
+        ExternalRTreeOpContext ctx = (ExternalRTreeOpContext) ictx;
+        List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
+
+        LSMRTreeCursorInitialState initialState = new LSMRTreeCursorInitialState(rtreeLeafFrameFactory,
+                rtreeInteriorFrameFactory, btreeLeafFrameFactory, ctx.getBTreeMultiComparator(), lsmHarness,
+                comparatorFields, linearizerArray, ctx.searchCallback, operationalComponents);
+
+        cursor.open(initialState, pred);
+    }
+
+    // The only reason for overriding the merge method is the way to determine
+    // the need to keep deleted tuples
+    // This can be done in a better way by creating a method boolean
+    // keepDeletedTuples(mergedComponents);
+    @Override
+    public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+        LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
+        ITreeIndexCursor cursor = mergeOp.getCursor();
+        ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
+        ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
+        opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
+        search(opCtx, cursor, rtreeSearchPred);
+
+        LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
+                mergeOp.getBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
+
+        // In case we must keep the deleted-keys BTrees, then they must be
+        // merged *before* merging the r-trees so that
+        // lsmHarness.endSearch() is called once when the r-trees have been
+        // merged.
+        if (mergeOp.isKeepDeletedTuples()) {
+            // Keep the deleted tuples since the oldest disk component is not
+            // included in the merge operation
+
+            LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx);
+            search(opCtx, btreeCursor, rtreeSearchPred);
+
+            BTree btree = mergedComponent.getBTree();
+            IIndexBulkLoader btreeBulkLoader = btree.createBulkLoader(1.0f, true, 0L, false);
+
+            long numElements = 0L;
+            for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+                numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
+                        .getNumElements();
+            }
+
+            int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
+            BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+                    bloomFilterFalsePositiveRate);
+            IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
+                    bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+
+            try {
+                while (btreeCursor.hasNext()) {
+                    btreeCursor.next();
+                    ITupleReference tuple = btreeCursor.getTuple();
+                    btreeBulkLoader.add(tuple);
+                    builder.add(tuple);
+                }
+            } finally {
+                btreeCursor.close();
+                builder.end();
+            }
+            btreeBulkLoader.end();
+        }
+
+        IIndexBulkLoader bulkLoader = mergedComponent.getRTree().createBulkLoader(1.0f, false, 0L, false);
+        try {
+            while (cursor.hasNext()) {
+                cursor.next();
+                ITupleReference frameTuple = cursor.getTuple();
+                bulkLoader.add(frameTuple);
+            }
+        } finally {
+            cursor.close();
+        }
+        bulkLoader.end();
+        return mergedComponent;
+    }
+
+    @Override
+    public void deactivate(boolean flushOnExit) throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
+        }
+
+        if (flushOnExit) {
+            BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
+            cb.afterFinalize(LSMOperationType.FLUSH, null);
+        }
+
+        for (ILSMComponent c : diskComponents) {
+            LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+            RTree rtree = component.getRTree();
+            BTree btree = component.getBTree();
+            BloomFilter bloomFilter = component.getBloomFilter();
+            rtree.deactivate();
+            btree.deactivate();
+            bloomFilter.deactivate();
+        }
+        for (ILSMComponent c : secondDiskComponents) {
+            // Only deactivate non shared components
+            if (!diskComponents.contains(c)) {
+                LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+                RTree rtree = component.getRTree();
+                BTree btree = component.getBTree();
+                BloomFilter bloomFilter = component.getBloomFilter();
+                rtree.deactivate();
+                btree.deactivate();
+                bloomFilter.deactivate();
+            }
+        }
+        isActivated = false;
+    }
+
+    // The clear method is not used anywhere in AsterixDB! we override it anyway
+    // to exit components first and clear the two lists
+    @Override
+    public void clear() throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to clear the index since it is not activated.");
+        }
+        ((ExternalIndexHarness) lsmHarness).indexClear();
+
+        for (ILSMComponent c : diskComponents) {
+            LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+            component.getRTree().deactivate();
+            component.getBloomFilter().deactivate();
+            component.getBTree().deactivate();
+            component.getRTree().destroy();
+            component.getBloomFilter().destroy();
+            component.getBTree().destroy();
+            // Remove from second list to avoid destroying twice
+            secondDiskComponents.remove(c);
+        }
+
+        for (ILSMComponent c : secondDiskComponents) {
+            LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+            component.getRTree().deactivate();
+            component.getBloomFilter().deactivate();
+            component.getBTree().deactivate();
+            component.getRTree().destroy();
+            component.getBloomFilter().destroy();
+            component.getBTree().destroy();
+        }
+
+        diskComponents.clear();
+        secondDiskComponents.clear();
+        version = -1;
+    }
+
+    @Override
+    public void destroy() throws HyracksDataException {
+        if (isActivated) {
+            throw new HyracksDataException("Failed to destroy the index since it is activated.");
+        }
+        for (ILSMComponent c : diskComponents) {
+            LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+            component.getRTree().destroy();
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
+            // Remove from second list to avoid destroying twice
+            secondDiskComponents.remove(c);
+        }
+        for (ILSMComponent c : secondDiskComponents) {
+            LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+            component.getRTree().destroy();
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
+        }
+        diskComponents.clear();
+        secondDiskComponents.clear();
+        fileManager.deleteDirs();
+        version = -1;
+    }
+
+    // Not supported
+    @Override
+    public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+        throw new UnsupportedOperationException("tuple modify not supported in LSM-Disk-Only-RTree");
+    }
+
+    // Not supported
+    @Override
+    public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException {
+        throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-RTree");
+    }
+
+    // Not supported
+    @Override
+    public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+        throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-RTree");
+    }
+
+    // Only support search and merge operations
+    @Override
+    public void getOperationalComponents(ILSMIndexOperationContext ctx) {
+        List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+        List<ILSMComponent> immutableComponents;
+        // Identify current list in case of a merge
+        if (version == 0) {
+            immutableComponents = diskComponents;
+        } else {
+            immutableComponents = secondDiskComponents;
+        }
+        ExternalRTreeOpContext opCtx = (ExternalRTreeOpContext) ctx;
+        operationalComponents.clear();
+        switch (ctx.getOperation()) {
+            case SEARCH:
+                if (opCtx.getTargetIndexVersion() == 0) {
+                    operationalComponents.addAll(diskComponents);
+                } else {
+                    operationalComponents.addAll(secondDiskComponents);
+                }
+                break;
+            case MERGE:
+                operationalComponents.addAll(ctx.getComponentsToBeMerged());
+                break;
+            case FULL_MERGE:
+                operationalComponents.addAll(immutableComponents);
+                break;
+            case FLUSH:
+                // Do nothing. this is left here even though the index never
+                // performs flushes because a flush is triggered by
+                // dataset lifecycle manager when closing an index. Having no
+                // components is a no operation
+                break;
+            default:
+                throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
+        }
+    }
+
+    // For initial load
+    @Override
+    public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex) throws TreeIndexException {
+        try {
+            return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex, false);
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    // For transaction bulk load <- could consolidate with the above method ->
+    @Override
+    public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex) throws TreeIndexException {
+        try {
+            return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, true);
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    // The bulk loader used for both initial loading and transaction
+    // modifications
+    public class LSMTwoPCRTreeBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
+        private final ILSMComponent component;
+        private final IIndexBulkLoader rtreeBulkLoader;
+        private final BTreeBulkLoader btreeBulkLoader;
+        private final IIndexBulkLoader builder;
+        private boolean cleanedUpArtifacts = false;
+        private boolean isEmptyComponent = true;
+        private boolean endedBloomFilterLoad = false;
+        private boolean isTransaction;
+
+        public LSMTwoPCRTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+                boolean checkIfEmptyIndex, boolean isTransaction) throws TreeIndexException, HyracksDataException {
+            this.isTransaction = isTransaction;
+            // Create the appropriate target
+            if (isTransaction) {
+                try {
+                    component = createTransactionTarget();
+                } catch (HyracksDataException | IndexException e) {
+                    throw new TreeIndexException(e);
+                }
+            } else {
+                if (checkIfEmptyIndex && !isEmptyIndex()) {
+                    throw new TreeIndexException("Cannot load an index that is not empty");
+                }
+                try {
+                    component = createBulkLoadTarget();
+                } catch (HyracksDataException | IndexException e) {
+                    throw new TreeIndexException(e);
+                }
+            }
+
+            // Create the three loaders
+            rtreeBulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
+                    numElementsHint, false);
+            btreeBulkLoader = (BTreeBulkLoader) ((LSMRTreeDiskComponent) component).getBTree().createBulkLoader(
+                    fillFactor, verifyInput, numElementsHint, false);
+            int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
+            BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+                    bloomFilterFalsePositiveRate);
+            builder = ((LSMRTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
+                    bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+        }
+
+        @Override
+        public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
+            try {
+                rtreeBulkLoader.add(tuple);
+            } catch (IndexException | HyracksDataException | RuntimeException e) {
+                cleanupArtifacts();
+                throw e;
+            }
+            if (isEmptyComponent) {
+                isEmptyComponent = false;
+            }
+        }
+
+        // This is made public in case of a failure, it is better to delete all
+        // created artifacts.
+        public void cleanupArtifacts() throws HyracksDataException {
+            if (!cleanedUpArtifacts) {
+                cleanedUpArtifacts = true;
+                try {
+                    ((LSMRTreeDiskComponent) component).getRTree().deactivate();
+                } catch (Exception e) {
+
+                }
+                ((LSMRTreeDiskComponent) component).getRTree().destroy();
+                try {
+                    ((LSMRTreeDiskComponent) component).getBTree().deactivate();
+                } catch (Exception e) {
+
+                }
+                ((LSMRTreeDiskComponent) component).getBTree().destroy();
+                try {
+                    ((LSMRTreeDiskComponent) component).getBloomFilter().deactivate();
+                } catch (Exception e) {
+
+                }
+                ((LSMRTreeDiskComponent) component).getBloomFilter().destroy();
+            }
+        }
+
+        @Override
+        public void end() throws HyracksDataException, IndexException {
+            if (!cleanedUpArtifacts) {
+                if (!endedBloomFilterLoad) {
+                    builder.end();
+                    endedBloomFilterLoad = true;
+                }
+                rtreeBulkLoader.end();
+                btreeBulkLoader.end();
+                if (isEmptyComponent) {
+                    cleanupArtifacts();
+                } else if (isTransaction) {
+                    // Since this is a transaction component, validate and
+                    // deactivate. it could later be added or deleted
+                    markAsValid(component);
+                    RTree rtree = ((LSMRTreeDiskComponent) component).getRTree();
+                    BTree btree = ((LSMRTreeDiskComponent) component).getBTree();
+                    BloomFilter bloomFilter = ((LSMRTreeDiskComponent) component).getBloomFilter();
+                    rtree.deactivate();
+                    btree.deactivate();
+                    bloomFilter.deactivate();
+                } else {
+                    lsmHarness.addBulkLoadedComponent(component);
+                }
+            }
+        }
+
+        @Override
+        public void delete(ITupleReference tuple) throws IndexException, HyracksDataException {
+            try {
+                btreeBulkLoader.add(tuple);
+                builder.add(tuple);
+            } catch (IndexException | HyracksDataException | RuntimeException e) {
+                cleanupArtifacts();
+                throw e;
+            }
+            if (isEmptyComponent) {
+                isEmptyComponent = false;
+            }
+        }
+
+        @Override
+        public void abort() {
+            try {
+                cleanupArtifacts();
+            } catch (Exception e) {
+
+            }
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "LSMTwoPCRTree [" + fileManager.getBaseDir() + "]";
+    }
+
+    // The only change the the schedule merge is the method used to create the
+    // opCtx. first line <- in schedule merge, we->
+    @Override
+    public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+            throws HyracksDataException, IndexException {
+        ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, -1);
+        rctx.setOperation(IndexOperation.MERGE);
+        List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+        ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
+        LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
+        ILSMIndexAccessorInternal accessor = new LSMRTreeAccessor(lsmHarness, rctx);
+        // create the merge operation.
+        LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation((ILSMIndexAccessorInternal) accessor,
+                mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(),
+                relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(),
+                callback, fileManager.getBaseDir());
+        // set the keepDeletedTuples flag
+        boolean keepDeleteTuples = false;
+        if (version == 0) {
+            keepDeleteTuples = mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents
+                    .get(diskComponents.size() - 1);
+        } else {
+            keepDeleteTuples = mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != secondDiskComponents
+                    .get(secondDiskComponents.size() - 1);
+        }
+        mergeOp.setKeepDeletedTuples(keepDeleteTuples);
+
+        ioScheduler.scheduleOperation(mergeOp);
+    }
+
+    @Override
+    public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
+            throws HyracksDataException {
+        return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
+    }
+
+    // This method creates the appropriate opContext for the targeted version
+    public ExternalRTreeOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
+        return new ExternalRTreeOpContext(rtreeCmpFactories, btreeCmpFactories, searchCallback, targetVersion);
+    }
+
+    // The accessor for disk only indexes don't use modification callback and
+    // always carry the target index version with them
+    @Override
+    public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+            ISearchOperationCallback searchCallback) {
+        return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, version));
+    }
+
+    public int getCurrentVersion() {
+        return version;
+    }
+
+    public List<ILSMComponent> getFirstComponentList() {
+        return diskComponents;
+    }
+
+    public List<ILSMComponent> getSecondComponentList() {
+        return secondDiskComponents;
+    }
+
+    @Override
+    public void commitTransaction() throws TreeIndexException, HyracksDataException, IndexException {
+        LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
+        LSMRTreeDiskComponent component = null;
+        if (componentFileRefrences != null) {
+            component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(),
+                    componentFileRefrences.getDeleteIndexFileReference(),
+                    componentFileRefrences.getBloomFilterFileReference(), false);
+        }
+        ((ExternalIndexHarness) lsmHarness).addTransactionComponents(component);
+    }
+
+    @Override
+    public void abortTransaction() throws TreeIndexException {
+        try {
+            fileManager.deleteTransactionFiles();
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    @Override
+    public void recoverTransaction() throws TreeIndexException {
+        try {
+            fileManager.recoverTransaction();
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+    
+    @Override
+    public boolean hasMemoryComponents(){
+        return false;
+    }
+    
+    @Override
+    public int getFieldCount() {
+        return fieldCount;
+    }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
new file mode 100644
index 0000000..55d0cad
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class ExternalRTreeOpContext implements ILSMIndexOperationContext {
+	private IndexOperation op;
+	private MultiComparator bTreeCmp;
+	private MultiComparator rTreeCmp;
+	public final List<ILSMComponent> componentHolder;
+	private final List<ILSMComponent> componentsToBeMerged;
+	public final ISearchOperationCallback searchCallback;
+	private final int targetIndexVersion;
+
+	public ExternalRTreeOpContext(
+			IBinaryComparatorFactory[] rtreeCmpFactories,
+			IBinaryComparatorFactory[] btreeCmpFactories,
+			ISearchOperationCallback searchCallback, int targetIndexVersion) {
+
+		this.componentHolder = new LinkedList<ILSMComponent>();
+		this.componentsToBeMerged = new LinkedList<ILSMComponent>();
+		this.searchCallback = searchCallback;
+		this.targetIndexVersion = targetIndexVersion;
+		this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
+		this.rTreeCmp = MultiComparator.create(rtreeCmpFactories);
+	}
+
+	public void setOperation(IndexOperation newOp) {
+		reset();
+		this.op = newOp;
+	}
+
+	@Override
+	public void setCurrentMutableComponentId(int currentMutableComponentId) {
+		// Do nothing. this should never be called for disk only indexes
+	}
+
+	@Override
+	public void reset() {
+		componentHolder.clear();
+		componentsToBeMerged.clear();
+	}
+
+	@Override
+	public IndexOperation getOperation() {
+		return op;
+	}
+
+	public MultiComparator getBTreeMultiComparator() {
+		return bTreeCmp;
+	}
+
+	public MultiComparator getRTreeMultiComparator() {
+		return rTreeCmp;
+	}
+
+	@Override
+	public List<ILSMComponent> getComponentHolder() {
+		return componentHolder;
+	}
+
+	@Override
+	public ISearchOperationCallback getSearchOperationCallback() {
+		return searchCallback;
+	}
+
+	// This should never be needed for disk only indexes
+	@Override
+	public IModificationOperationCallback getModificationCallback() {
+		return null;
+	}
+
+	@Override
+	public List<ILSMComponent> getComponentsToBeMerged() {
+		return componentsToBeMerged;
+	}
+
+	public int getTargetIndexVersion() {
+		return targetIndexVersion;
+	}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 0d35ab3..37ce427 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -66,7 +66,7 @@
 import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMRTree extends AbstractLSMRTree {
-    private int[] buddyBTreeFields;
+    protected int[] buddyBTreeFields;
 
     public LSMRTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
             ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
@@ -85,6 +85,26 @@
                 mergePolicy, opTracker, ioScheduler, ioOpCallback);
         this.buddyBTreeFields = buddyBTreeFields;
     }
+    
+    /*
+     * For External indexes with no memory components
+     */
+    public LSMRTree(ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+            ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
+            ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory,
+            TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory,
+            double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+            IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+            ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
+            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields) {
+        super(rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
+                fileNameManager, new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory,
+                        bloomFilterFactory), diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories,
+                linearizer, comparatorFields, linearizerArray, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
+                ioScheduler, ioOpCallback);
+        this.buddyBTreeFields = buddyBTreeFields;
+    }
 
     /**
      * Opens LSMRTree, cleaning up invalid files from base dir, and registering
@@ -405,7 +425,7 @@
         }
     }
 
-    private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+    protected ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
         LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
         return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
                 componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 9502d5e..cca640f 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -17,6 +17,9 @@
 
 import java.io.File;
 import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -77,7 +80,8 @@
         String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
 
         String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
-        // Get the range of timestamps by taking the earliest and the latest timestamps
+        // Get the range of timestamps by taking the earliest and the latest
+        // timestamps
         return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + RTREE_STRING),
                 createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), createMergeFile(baseName + SPLIT_STRING
                         + BLOOM_FILTER_STRING));
@@ -90,15 +94,18 @@
         ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
         ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
 
+        // Create a transaction filter <- to hide transaction components->
+        FilenameFilter transactionFilter = getTransactionFileFilter(false);
+
         // Gather files.
-        cleanupAndGetValidFilesInternal(btreeFilter, btreeFactory, allBTreeFiles);
+        cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
         HashSet<String> btreeFilesSet = new HashSet<String>();
         for (ComparableFileName cmpFileName : allBTreeFiles) {
             int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
             btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
         }
-        validateFiles(btreeFilesSet, allRTreeFiles, rtreeFilter, rtreeFactory);
-        validateFiles(btreeFilesSet, allBloomFilterFiles, bloomFilterFilter, null);
+        validateFiles(btreeFilesSet, allRTreeFiles, getCompoundFilter(transactionFilter, rtreeFilter), rtreeFactory);
+        validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(transactionFilter, bloomFilterFilter), null);
 
         // Sanity check.
         if (allRTreeFiles.size() != allBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) {
@@ -183,7 +190,62 @@
             validFiles.add(new LSMComponentFileReferences(cmpRTreeFileName.fileRef, cmpBTreeFileName.fileRef,
                     cmpBloomFilterFileName.fileRef));
         }
-
         return validFiles;
     }
+
+    @Override
+    public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
+        Date date = new Date();
+        String ts = formatter.format(date);
+        // Create transaction lock file
+        Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
+
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + RTREE_STRING),
+                createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
+                        + BLOOM_FILTER_STRING));
+    }
+
+    @Override
+    public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
+        FilenameFilter transactionFilter;
+        File dir = new File(baseDir);
+        String[] files = dir.list(transactionFileNameFilter);
+        if (files.length == 0)
+            return null;
+        if (files.length != 1) {
+            throw new HyracksDataException("More than one transaction lock found:" + files.length);
+        } else {
+            transactionFilter = getTransactionFileFilter(true);
+            String txnFileName = dir.getPath() + File.separator + files[0];
+            // get the actual transaction files
+            files = dir.list(transactionFilter);
+            if (files.length < 3) {
+                throw new HyracksDataException("LSM Rtree transaction has less than 3 files :" + files.length);
+            }
+            try {
+                Files.delete(Paths.get(txnFileName));
+            } catch (IOException e) {
+                throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName);
+            }
+        }
+        File rTreeFile = null;
+        File bTreeFile = null;
+        File bloomFilterFile = null;
+        for (String fileName : files) {
+            if (fileName.endsWith(BTREE_STRING)) {
+                bTreeFile = new File(dir.getPath() + File.separator + fileName);
+            } else if (fileName.endsWith(RTREE_STRING)) {
+                rTreeFile = new File(dir.getPath() + File.separator + fileName);
+            } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
+                bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
+            } else {
+                throw new HyracksDataException("unrecognized file found = " + fileName);
+            }
+        }
+        FileReference rTreeFileRef = new FileReference(rTreeFile);
+        FileReference bTreeFileRef = new FileReference(bTreeFile);
+        FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+        return new LSMComponentFileReferences(rTreeFileRef, bTreeFileRef, bloomFilterFileRef);
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
index ddf3ebf..4b94600 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
@@ -37,6 +37,7 @@
     private final FileReference bloomFilterMergeTarget;
     private final ILSMIOOperationCallback callback;
     private final String indexIdentifier;
+    private boolean keepDeletedTuples;
 
     public LSMRTreeMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents,
             ITreeIndexCursor cursor, FileReference rtreeMergeTarget, FileReference btreeMergeTarget,
@@ -116,4 +117,12 @@
     public LSMIOOpertionType getIOOpertionType() {
         return LSMIOOpertionType.MERGE;
     }
+
+    public boolean isKeepDeletedTuples() {
+        return keepDeletedTuples;
+    }
+
+    public void setKeepDeletedTuples(boolean keepDeletedTuples) {
+        this.keepDeletedTuples = keepDeletedTuples;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 13ecc84..224e449 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -41,6 +41,7 @@
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
 import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.ExternalRTree;
 import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
 import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager;
 import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples;
@@ -168,6 +169,62 @@
         return lsmTree;
     }
 
+    public static ExternalRTree createExternalRTree(FileReference file, IBufferCache diskBufferCache,
+            IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+            IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+            IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
+            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            ILinearizeComparatorFactory linearizeCmpFactory, int[] buddyBTreeFields, int startWithVersion)
+            throws TreeIndexException {
+
+        ITypeTraits[] btreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
+        for (int i = 0; i < btreeTypeTraits.length; i++) {
+            btreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
+        }
+
+        LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+        LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(btreeTypeTraits,
+                true);
+
+        ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
+                valueProviderFactories, rtreePolicyType);
+        ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeTupleWriterFactory,
+                valueProviderFactories, rtreePolicyType);
+
+        ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
+        ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
+
+        ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+        LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+                metaFrameFactory);
+
+        TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
+                typeTraits.length);
+        TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+                freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories,
+                btreeTypeTraits.length);
+        int[] comparatorFields = { 0 };
+        IBinaryComparatorFactory[] linearizerArray = { linearizeCmpFactory };
+
+        int[] bloomFilterKeyFields = new int[btreeCmpFactories.length];
+        for (int i = 0; i < btreeCmpFactories.length; i++) {
+            bloomFilterKeyFields[i] = i;
+        }
+        BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
+                bloomFilterKeyFields);
+
+        ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(diskFileMapProvider, file, diskRTreeFactory,
+                diskBTreeFactory);
+        ExternalRTree lsmTree = new ExternalRTree(rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+                btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
+                bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length,
+                rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray,
+                mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, startWithVersion);
+        return lsmTree;
+    }
+
     public static ILinearizeComparatorFactory proposeBestLinearizer(ITypeTraits[] typeTraits, int numKeyFields)
             throws TreeIndexException {
         for (int i = 0; i < numKeyFields; i++) {
diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
index d91e02e..dd69e67 100644
--- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
+++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
@@ -28,6 +28,9 @@
     public static final int LSMBTreeResource = 1;
     public static final int LSMRTreeResource = 2;
     public static final int LSMInvertedIndexResource = 3;
+    public static final int ExternalBTreeResource = 4;
+    public static final int ExternalRTreeResource = 5;
+    public static final int ExternalBTreeWithBuddyResource = 6;
 
     public LocalResource(long resourceId, String resourceName, int partition, int resourceType, Object object) {
         this.resourceId = resourceId;